diff --git a/.gitattributes b/.gitattributes
index e7557fab207778c4ec1497401e30e3c83c454c5c..857b422b0ed5ffae087d4f2d1d008c1b4cf004ce 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -64,3 +64,20 @@ qwen3/fdd_sfkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 qwen3/fdd_sfkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 qwen3/fdd_sfkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 qwen3/fdd_srkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fdd_srkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fdd_srkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fdd_srkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fdd_srkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fdd_srkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/fkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/hpd/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/hpd/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/hpd/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/hpd/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/hpd/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/hpd/tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/qwen3/fdd_srkl/checkpoint-1570/README.md b/qwen3/fdd_srkl/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-1570/adapter_config.json b/qwen3/fdd_srkl/checkpoint-1570/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b3d78327098bdde7084c1184d3568542c0fc45b4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-1570/adapter_model.safetensors b/qwen3/fdd_srkl/checkpoint-1570/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..aaf004a5282d6eff939542541b017b669c2028af
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99fc0546ccfaf4ac4d14c8f45aee81602af1b5a2055a86913c4740cf4b7ef42c
+size 40422560
diff --git a/qwen3/fdd_srkl/checkpoint-1570/chat_template.jinja b/qwen3/fdd_srkl/checkpoint-1570/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33cb74d90390a1a6b94e8184d5fe327a8923dd13
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a856061b9a2b9c3ec1266cf51ede9955d8cd358bca34cc9690f7d8c9571b13
+size 121135557
diff --git a/qwen3/fdd_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6077173e56173554570b70e87fdc689c30748824
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd3ed025c90f3c11c7cc82b06136ff01acb8b31680f7561a1025b3f92f4e09fe
+size 121135749
diff --git a/qwen3/fdd_srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/fdd_srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..17f9400f448a778c6eab89519f00c7082872d4c0
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2da22aa2281ba02d5a68f3846200d692b6adf5f124f93cc2389e651bdfd59fca
+size 351841765
diff --git a/qwen3/fdd_srkl/checkpoint-1570/latest b/qwen3/fdd_srkl/checkpoint-1570/latest
new file mode 100644
index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/latest
@@ -0,0 +1 @@
+global_step1570
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-1570/rng_state_0.pth b/qwen3/fdd_srkl/checkpoint-1570/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7ade87dbb295e3f7a337ae66a526cf6d3614b224
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aecd1c085596d7db97dd4b4f44c436f9a4f4b006b7b4918ad4eff80706a73c3
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-1570/rng_state_1.pth b/qwen3/fdd_srkl/checkpoint-1570/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6b828840e6a4a30b5ccacdb187b0dfd5351608a8
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5806b5397c5b16eef73bdab6d9dab79e97b62c7104e2b89e5d27d753bcd0057a
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-1570/scheduler.pt b/qwen3/fdd_srkl/checkpoint-1570/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae
+size 1465
diff --git a/qwen3/fdd_srkl/checkpoint-1570/tokenizer.json b/qwen3/fdd_srkl/checkpoint-1570/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fdd_srkl/checkpoint-1570/tokenizer_config.json b/qwen3/fdd_srkl/checkpoint-1570/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-1570/trainer_state.json b/qwen3/fdd_srkl/checkpoint-1570/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..4a51166fe6f90070bc5053e0cf00879dfdec334d
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/trainer_state.json
@@ -0,0 +1,655 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 1570,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.4362361431121826,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 1.4636783599853516,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 2.9437320232391357,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.5306460380554199,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 1.3007458448410034,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.34657702445983884,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 1.569557547569275,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.25293736457824706,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.8431988954544067,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.208502197265625,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 2.0384669303894043,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.16615729331970214,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 1.341955542564392,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.14722728729248047,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.9276875257492065,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.14779376983642578,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 1.3371649980545044,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.13147571086883544,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 1.0184792280197144,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.14180898666381836,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.5344496369361877,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.12216286659240723,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 3.0611512660980225,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.11602286100387574,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.5453760027885437,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.11221114397048951,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.4289671778678894,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.11079610586166382,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.4820654094219208,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.10537848472595215,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 87.9089,
+      "eval_generator_rouge1": 99.464,
+      "eval_generator_rouge2": 99.1899,
+      "eval_generator_rougeL": 99.0517,
+      "eval_loss": 0.009097261354327202,
+      "eval_runtime": 606.4571,
+      "eval_samples_per_second": 5.629,
+      "eval_selector_accuracy": 96.6706,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.176,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 3.0870883464813232,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.09972293376922607,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.7636368870735168,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.08893377780914306,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.4203242063522339,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.08573378920555115,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.371820867061615,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.07898830771446227,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.676804780960083,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.08897324800491332,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 1.0131059885025024,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.08602790236473083,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.4733346104621887,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.08213704824447632,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.9082027673721313,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.0822598934173584,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.3575717508792877,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0718361496925354,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.3527005612850189,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.07293112277984619,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.3359999358654022,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.07311422824859619,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.3345245122909546,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.07376066446304322,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.3286590874195099,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.06656086444854736,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.41298067569732666,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.07345538735389709,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2776452600955963,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.06613650321960449,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.27869823575019836,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.06492968797683715,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 94.743,
+      "eval_generator_rouge1": 99.8867,
+      "eval_generator_rouge2": 99.7804,
+      "eval_generator_rougeL": 99.6736,
+      "eval_loss": 0.006523935589939356,
+      "eval_runtime": 602.2662,
+      "eval_samples_per_second": 5.669,
+      "eval_selector_accuracy": 97.3715,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.178,
+      "step": 628
+    },
+    {
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.5264983177185059,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.06454238891601563,
+      "step": 640
+    },
+    {
+      "epoch": 2.102114080574392,
+      "grad_norm": 0.7168541550636292,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.06528459787368775,
+      "step": 660
+    },
+    {
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.4798216223716736,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.06159168481826782,
+      "step": 680
+    },
+    {
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.5430735349655151,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.06343585252761841,
+      "step": 700
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.2429123818874359,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.06073571443557739,
+      "step": 720
+    },
+    {
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.2920048236846924,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.05954910516738891,
+      "step": 740
+    },
+    {
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.19888697564601898,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.05689072012901306,
+      "step": 760
+    },
+    {
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.3021155893802643,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0560643196105957,
+      "step": 780
+    },
+    {
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.32512202858924866,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.0600568413734436,
+      "step": 800
+    },
+    {
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.53708815574646,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.05846037864685059,
+      "step": 820
+    },
+    {
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.3181110620498657,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.05601181983947754,
+      "step": 840
+    },
+    {
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.22879371047019958,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.05564823150634766,
+      "step": 860
+    },
+    {
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.24975994229316711,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.05476831197738648,
+      "step": 880
+    },
+    {
+      "epoch": 2.8679696848823295,
+      "grad_norm": 0.3306073546409607,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.05817422866821289,
+      "step": 900
+    },
+    {
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.1665683090686798,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.05767209529876709,
+      "step": 920
+    },
+    {
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.3881584703922272,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.05669119954109192,
+      "step": 940
+    },
+    {
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.0935,
+      "eval_generator_rouge1": 99.8705,
+      "eval_generator_rouge2": 99.7886,
+      "eval_generator_rougeL": 99.6664,
+      "eval_loss": 0.005655109416693449,
+      "eval_runtime": 604.2347,
+      "eval_samples_per_second": 5.65,
+      "eval_selector_accuracy": 98.4813,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.177,
+      "step": 942
+    },
+    {
+      "epoch": 3.057439170323095,
+      "grad_norm": 0.2090013325214386,
+      "learning_rate": 3.3007844105971336e-05,
+      "loss": 0.05627270340919495,
+      "step": 960
+    },
+    {
+      "epoch": 3.12126047068209,
+      "grad_norm": 0.2156447023153305,
+      "learning_rate": 3.114292593987149e-05,
+      "loss": 0.05334089994430542,
+      "step": 980
+    },
+    {
+      "epoch": 3.185081771041085,
+      "grad_norm": 0.1974238008260727,
+      "learning_rate": 2.9308285746383408e-05,
+      "loss": 0.051919639110565186,
+      "step": 1000
+    },
+    {
+      "epoch": 3.24890307140008,
+      "grad_norm": 0.18332256376743317,
+      "learning_rate": 2.7506861536187867e-05,
+      "loss": 0.05656099319458008,
+      "step": 1020
+    },
+    {
+      "epoch": 3.3127243717590744,
+      "grad_norm": 0.7237991094589233,
+      "learning_rate": 2.5741538127566396e-05,
+      "loss": 0.05259966850280762,
+      "step": 1040
+    },
+    {
+      "epoch": 3.3765456721180693,
+      "grad_norm": 0.2267737239599228,
+      "learning_rate": 2.4015142526625115e-05,
+      "loss": 0.05074975490570068,
+      "step": 1060
+    },
+    {
+      "epoch": 3.4403669724770642,
+      "grad_norm": 0.20692221820354462,
+      "learning_rate": 2.233043940009967e-05,
+      "loss": 0.054996371269226074,
+      "step": 1080
+    },
+    {
+      "epoch": 3.504188272836059,
+      "grad_norm": 0.15545181930065155,
+      "learning_rate": 2.0690126647990974e-05,
+      "loss": 0.05093409419059754,
+      "step": 1100
+    },
+    {
+      "epoch": 3.568009573195054,
+      "grad_norm": 0.1771135926246643,
+      "learning_rate": 1.9096831083121828e-05,
+      "loss": 0.05136837959289551,
+      "step": 1120
+    },
+    {
+      "epoch": 3.6318308735540485,
+      "grad_norm": 0.24874085187911987,
+      "learning_rate": 1.7553104224533324e-05,
+      "loss": 0.05098183155059814,
+      "step": 1140
+    },
+    {
+      "epoch": 3.6956521739130435,
+      "grad_norm": 0.17957162857055664,
+      "learning_rate": 1.606141821145735e-05,
+      "loss": 0.050333261489868164,
+      "step": 1160
+    },
+    {
+      "epoch": 3.7594734742720384,
+      "grad_norm": 0.1954505741596222,
+      "learning_rate": 1.4624161844408763e-05,
+      "loss": 0.05178115367889404,
+      "step": 1180
+    },
+    {
+      "epoch": 3.823294774631033,
+      "grad_norm": 0.19122923910617828,
+      "learning_rate": 1.324363675973698e-05,
+      "loss": 0.050999945402145384,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8871160749900278,
+      "grad_norm": 0.26517826318740845,
+      "learning_rate": 1.1922053743763052e-05,
+      "loss": 0.0516182005405426,
+      "step": 1220
+    },
+    {
+      "epoch": 3.9509373753490227,
+      "grad_norm": 0.1767585426568985,
+      "learning_rate": 1.0661529192404973e-05,
+      "loss": 0.04989046454429626,
+      "step": 1240
+    },
+    {
+      "epoch": 4.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.3201,
+      "eval_generator_rouge1": 99.9029,
+      "eval_generator_rouge2": 99.8325,
+      "eval_generator_rougeL": 99.7566,
+      "eval_loss": 0.0041970605961978436,
+      "eval_runtime": 743.2973,
+      "eval_samples_per_second": 4.593,
+      "eval_selector_accuracy": 98.5397,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.144,
+      "step": 1256
+    },
+    {
+      "epoch": 4.0127642600717985,
+      "grad_norm": 0.17936867475509644,
+      "learning_rate": 9.464081721960592e-06,
+      "loss": 0.05013519525527954,
+      "step": 1260
+    },
+    {
+      "epoch": 4.076585560430794,
+      "grad_norm": 0.323055237531662,
+      "learning_rate": 8.331628936475818e-06,
+      "loss": 0.04865485727787018,
+      "step": 1280
+    },
+    {
+      "epoch": 4.140406860789788,
+      "grad_norm": 0.16135305166244507,
+      "learning_rate": 7.2659843568748385e-06,
+      "loss": 0.04849388599395752,
+      "step": 1300
+    },
+    {
+      "epoch": 4.204228161148784,
+      "grad_norm": 0.17001892626285553,
+      "learning_rate": 6.268854516770008e-06,
+      "loss": 0.04871906638145447,
+      "step": 1320
+    },
+    {
+      "epoch": 4.268049461507778,
+      "grad_norm": 0.18477433919906616,
+      "learning_rate": 5.341836229602331e-06,
+      "loss": 0.048744958639144895,
+      "step": 1340
+    },
+    {
+      "epoch": 4.331870761866773,
+      "grad_norm": 0.1783062219619751,
+      "learning_rate": 4.4864140314888216e-06,
+      "loss": 0.04793646335601807,
+      "step": 1360
+    },
+    {
+      "epoch": 4.395692062225768,
+      "grad_norm": 0.17276793718338013,
+      "learning_rate": 3.703957803871932e-06,
+      "loss": 0.048929840326309204,
+      "step": 1380
+    },
+    {
+      "epoch": 4.4595133625847625,
+      "grad_norm": 0.1627519130706787,
+      "learning_rate": 2.9957205797780495e-06,
+      "loss": 0.04951263666152954,
+      "step": 1400
+    },
+    {
+      "epoch": 4.523334662943758,
+      "grad_norm": 0.28899943828582764,
+      "learning_rate": 2.362836537198199e-06,
+      "loss": 0.05062735080718994,
+      "step": 1420
+    },
+    {
+      "epoch": 4.587155963302752,
+      "grad_norm": 0.17383745312690735,
+      "learning_rate": 1.8063191828043703e-06,
+      "loss": 0.049233487248420714,
+      "step": 1440
+    },
+    {
+      "epoch": 4.650977263661747,
+      "grad_norm": 0.15494407713413239,
+      "learning_rate": 1.3270597289100437e-06,
+      "loss": 0.04810199737548828,
+      "step": 1460
+    },
+    {
+      "epoch": 4.714798564020742,
+      "grad_norm": 0.14835470914840698,
+      "learning_rate": 9.258256662741205e-07,
+      "loss": 0.04752459526062012,
+      "step": 1480
+    },
+    {
+      "epoch": 4.778619864379737,
+      "grad_norm": 0.17123174667358398,
+      "learning_rate": 6.032595350337341e-07,
+      "loss": 0.04880803823471069,
+      "step": 1500
+    },
+    {
+      "epoch": 4.842441164738732,
+      "grad_norm": 0.15825852751731873,
+      "learning_rate": 3.5987789573421356e-07,
+      "loss": 0.05428855419158936,
+      "step": 1520
+    },
+    {
+      "epoch": 4.9062624650977265,
+      "grad_norm": 1.0737415552139282,
+      "learning_rate": 1.9607050210401192e-07,
+      "loss": 0.0523545503616333,
+      "step": 1540
+    },
+    {
+      "epoch": 4.970083765456721,
+      "grad_norm": 0.18025070428848267,
+      "learning_rate": 1.1209967689925832e-07,
+      "loss": 0.049212086200714114,
+      "step": 1560
+    },
+    {
+      "epoch": 5.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.7874,
+      "eval_generator_rouge1": 99.9018,
+      "eval_generator_rouge2": 99.8253,
+      "eval_generator_rougeL": 99.7687,
+      "eval_loss": 0.0041342261247336864,
+      "eval_runtime": 612.0487,
+      "eval_samples_per_second": 5.578,
+      "eval_selector_accuracy": 98.8318,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 1570
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.871444840310374e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-1570/training_args.bin b/qwen3/fdd_srkl/checkpoint-1570/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..063309915a275e078e6f834ef9c384b1cb1c965e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb0e76106b10de1344a3493c5f04dde38b30a16fde01bc76445c98b578802418
+size 7185
diff --git a/qwen3/fdd_srkl/checkpoint-1570/zero_to_fp32.py b/qwen3/fdd_srkl/checkpoint-1570/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-1570/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fdd_srkl/checkpoint-314/README.md b/qwen3/fdd_srkl/checkpoint-314/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-314/adapter_config.json b/qwen3/fdd_srkl/checkpoint-314/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b3d78327098bdde7084c1184d3568542c0fc45b4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-314/adapter_model.safetensors b/qwen3/fdd_srkl/checkpoint-314/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cde36adb583d8de99ffe26d0cef9dd2d64c48fe4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0541597a8d58416bfd8df40ff0098eff9b6d1bd1dfd724a82bc5e004f4339abb
+size 40422560
diff --git a/qwen3/fdd_srkl/checkpoint-314/chat_template.jinja b/qwen3/fdd_srkl/checkpoint-314/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2769b60b9212739cbccd8c976371abf15d2967b
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e62f46f4e76b52e92eb30268f602e719efb9844b7407144a2dee0ac86a35e121
+size 121135557
diff --git a/qwen3/fdd_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fefd948ef6edff210f5a751cf66231c505ded719
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4798f49fd36837bdbcf0cedc60887464a3563bba6eea3ed356e2b487e185d668
+size 121135749
diff --git a/qwen3/fdd_srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/fdd_srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5309dc27f24f3dc7d3e97dde89de90d5013c1cb
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac2495f0251c136a218bc14d62f4a12ecec6f1b7406ef3ac3aef790c17e73f91
+size 351841765
diff --git a/qwen3/fdd_srkl/checkpoint-314/latest b/qwen3/fdd_srkl/checkpoint-314/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/latest
@@ -0,0 +1 @@
+global_step314
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-314/rng_state_0.pth b/qwen3/fdd_srkl/checkpoint-314/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..26e064e30b45cc93e45480feec509cd41557bfd3
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c58fa50d0eb2b9d0c41c4c950a755054f17af94fbb60383eee5e46d96797b4cd
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-314/rng_state_1.pth b/qwen3/fdd_srkl/checkpoint-314/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..9d3d426bd9b5bf50e3a60467650048b9c0db604d
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e89a8da6898755602708e3be195153442a313c7b82548f2de733720343c1812e
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-314/scheduler.pt b/qwen3/fdd_srkl/checkpoint-314/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149
+size 1465
diff --git a/qwen3/fdd_srkl/checkpoint-314/tokenizer.json b/qwen3/fdd_srkl/checkpoint-314/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fdd_srkl/checkpoint-314/tokenizer_config.json b/qwen3/fdd_srkl/checkpoint-314/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-314/trainer_state.json b/qwen3/fdd_srkl/checkpoint-314/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..87bb2a2d7b82966196fdb72d408a4cdebe2825de
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/trainer_state.json
@@ -0,0 +1,154 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 314,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.4362361431121826,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 1.4636783599853516,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 2.9437320232391357,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.5306460380554199,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 1.3007458448410034,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.34657702445983884,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 1.569557547569275,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.25293736457824706,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.8431988954544067,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.208502197265625,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 2.0384669303894043,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.16615729331970214,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 1.341955542564392,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.14722728729248047,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.9276875257492065,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.14779376983642578,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 1.3371649980545044,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.13147571086883544,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 1.0184792280197144,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.14180898666381836,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.5344496369361877,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.12216286659240723,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 3.0611512660980225,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.11602286100387574,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.5453760027885437,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.11221114397048951,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.4289671778678894,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.11079610586166382,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.4820654094219208,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.10537848472595215,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 87.9089,
+      "eval_generator_rouge1": 99.464,
+      "eval_generator_rouge2": 99.1899,
+      "eval_generator_rougeL": 99.0517,
+      "eval_loss": 0.009097261354327202,
+      "eval_runtime": 606.4571,
+      "eval_samples_per_second": 5.629,
+      "eval_selector_accuracy": 96.6706,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.176,
+      "step": 314
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.172842003562496e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-314/training_args.bin b/qwen3/fdd_srkl/checkpoint-314/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..063309915a275e078e6f834ef9c384b1cb1c965e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb0e76106b10de1344a3493c5f04dde38b30a16fde01bc76445c98b578802418
+size 7185
diff --git a/qwen3/fdd_srkl/checkpoint-314/zero_to_fp32.py b/qwen3/fdd_srkl/checkpoint-314/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-314/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fdd_srkl/checkpoint-628/README.md b/qwen3/fdd_srkl/checkpoint-628/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-628/adapter_config.json b/qwen3/fdd_srkl/checkpoint-628/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b3d78327098bdde7084c1184d3568542c0fc45b4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-628/adapter_model.safetensors b/qwen3/fdd_srkl/checkpoint-628/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..51c5123e5fcfa3a669f07eb82e73659ee4a82fd5
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c60a45c15516253317a55bd9d5a0cae46c3e4650e61ee00a1c7b18769d0724c
+size 40422560
diff --git a/qwen3/fdd_srkl/checkpoint-628/chat_template.jinja b/qwen3/fdd_srkl/checkpoint-628/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b74fcb0c8e52a02c1c4e85d956b9eb27fcf48e2a
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b129c1b9f8ce13480079741c6f238ab6b9c3dde27f44d00f26215716830ea0ab
+size 121135557
diff --git a/qwen3/fdd_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c47220a1bc826b398df1b63ed6c1f0041e9640a
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21a8ae54bdf1cbe89e467069dc29bbb07b9efb6623c0534f7f37c34b04efd6a9
+size 121135749
diff --git a/qwen3/fdd_srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/fdd_srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6bf851e398ff1d310e5929fe4a041d802b20e21d
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da73dee48711c1760c1f3486c6990db7982d1b81e97c9012bbd162967a7a91ea
+size 351841765
diff --git a/qwen3/fdd_srkl/checkpoint-628/latest b/qwen3/fdd_srkl/checkpoint-628/latest
new file mode 100644
index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/latest
@@ -0,0 +1 @@
+global_step628
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-628/rng_state_0.pth b/qwen3/fdd_srkl/checkpoint-628/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..9b771f0181d983c91f5d0ae023705992a728fc53
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0753776e10d2844af8605bc24a224af48189614693939c0c98ad8fef5a0e0b4
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-628/rng_state_1.pth b/qwen3/fdd_srkl/checkpoint-628/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d9dc404a82e957acf6c86951949df5224389bb86
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4978b524e78ad211507c5eca2a017bd220e731f0ada7a2f3126361d8f9286ad
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-628/scheduler.pt b/qwen3/fdd_srkl/checkpoint-628/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a
+size 1465
diff --git a/qwen3/fdd_srkl/checkpoint-628/tokenizer.json b/qwen3/fdd_srkl/checkpoint-628/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fdd_srkl/checkpoint-628/tokenizer_config.json b/qwen3/fdd_srkl/checkpoint-628/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-628/trainer_state.json b/qwen3/fdd_srkl/checkpoint-628/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..b2490ed58fcd53473191eec93260bdcb8dbdf86d
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/trainer_state.json
@@ -0,0 +1,281 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 628,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.4362361431121826,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 1.4636783599853516,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 2.9437320232391357,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.5306460380554199,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 1.3007458448410034,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.34657702445983884,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 1.569557547569275,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.25293736457824706,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.8431988954544067,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.208502197265625,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 2.0384669303894043,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.16615729331970214,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 1.341955542564392,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.14722728729248047,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.9276875257492065,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.14779376983642578,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 1.3371649980545044,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.13147571086883544,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 1.0184792280197144,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.14180898666381836,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.5344496369361877,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.12216286659240723,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 3.0611512660980225,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.11602286100387574,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.5453760027885437,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.11221114397048951,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.4289671778678894,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.11079610586166382,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.4820654094219208,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.10537848472595215,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 87.9089,
+      "eval_generator_rouge1": 99.464,
+      "eval_generator_rouge2": 99.1899,
+      "eval_generator_rougeL": 99.0517,
+      "eval_loss": 0.009097261354327202,
+      "eval_runtime": 606.4571,
+      "eval_samples_per_second": 5.629,
+      "eval_selector_accuracy": 96.6706,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.176,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 3.0870883464813232,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.09972293376922607,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.7636368870735168,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.08893377780914306,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.4203242063522339,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.08573378920555115,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.371820867061615,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.07898830771446227,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.676804780960083,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.08897324800491332,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 1.0131059885025024,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.08602790236473083,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.4733346104621887,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.08213704824447632,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.9082027673721313,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.0822598934173584,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.3575717508792877,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0718361496925354,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.3527005612850189,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.07293112277984619,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.3359999358654022,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.07311422824859619,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.3345245122909546,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.07376066446304322,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.3286590874195099,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.06656086444854736,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.41298067569732666,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.07345538735389709,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2776452600955963,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.06613650321960449,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.27869823575019836,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.06492968797683715,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 94.743,
+      "eval_generator_rouge1": 99.8867,
+      "eval_generator_rouge2": 99.7804,
+      "eval_generator_rougeL": 99.6736,
+      "eval_loss": 0.006523935589939356,
+      "eval_runtime": 602.2662,
+      "eval_samples_per_second": 5.669,
+      "eval_selector_accuracy": 97.3715,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.178,
+      "step": 628
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.346351770520781e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-628/training_args.bin b/qwen3/fdd_srkl/checkpoint-628/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..063309915a275e078e6f834ef9c384b1cb1c965e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb0e76106b10de1344a3493c5f04dde38b30a16fde01bc76445c98b578802418
+size 7185
diff --git a/qwen3/fdd_srkl/checkpoint-628/zero_to_fp32.py b/qwen3/fdd_srkl/checkpoint-628/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-628/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fdd_srkl/checkpoint-942/README.md b/qwen3/fdd_srkl/checkpoint-942/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-942/adapter_config.json b/qwen3/fdd_srkl/checkpoint-942/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b3d78327098bdde7084c1184d3568542c0fc45b4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-942/adapter_model.safetensors b/qwen3/fdd_srkl/checkpoint-942/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..386936832cf2f886777389868a574d9fbc69bfc5
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:974a9237b3cbb2d8f01b986ff86a2a78e758f1b8be6066847f1b78b85d5d5209
+size 40422560
diff --git a/qwen3/fdd_srkl/checkpoint-942/chat_template.jinja b/qwen3/fdd_srkl/checkpoint-942/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56a23caf8b3c59653effbadb04f34e2c201ad584
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e732167b5bb909ebb1d44d2fea0cf34a49cd5f48bcfa023093385d0cae8c6b4
+size 121135557
diff --git a/qwen3/fdd_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46a27a45e4b98b2137109a4c32fdcfc4df0ed4de
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d84ed3bd0f23ff1a4491ab48afc1edc3fbe2e5efac93df808d08c49a3bef06ee
+size 121135749
diff --git a/qwen3/fdd_srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/fdd_srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..72e8059bf069e087eb1ca76724bd3c7b1df59dc8
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:796d3aef90c212dcd1e965d17be1c975b3814bb0a16a82cbd1571604658a3759
+size 351841765
diff --git a/qwen3/fdd_srkl/checkpoint-942/latest b/qwen3/fdd_srkl/checkpoint-942/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/latest
@@ -0,0 +1 @@
+global_step942
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/checkpoint-942/rng_state_0.pth b/qwen3/fdd_srkl/checkpoint-942/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6ead818eb416487a8c36085ce4998b3d598fb697
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ee27c6e5aa920b6eafafc58bfccc7ce14265e09460c452f00c20808dde539c5
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-942/rng_state_1.pth b/qwen3/fdd_srkl/checkpoint-942/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e77de7b8935b0ddea00bebcef3fa6593936dc049
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da46a563fa8a9ef072534444e24e59e482024280c71eb0fa8e9181a6eab8668a
+size 14917
diff --git a/qwen3/fdd_srkl/checkpoint-942/scheduler.pt b/qwen3/fdd_srkl/checkpoint-942/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3
+size 1465
diff --git a/qwen3/fdd_srkl/checkpoint-942/tokenizer.json b/qwen3/fdd_srkl/checkpoint-942/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fdd_srkl/checkpoint-942/tokenizer_config.json b/qwen3/fdd_srkl/checkpoint-942/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-942/trainer_state.json b/qwen3/fdd_srkl/checkpoint-942/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..abbaa3142a5a59f46db0b30462e14eb6671c916b
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/trainer_state.json
@@ -0,0 +1,408 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 942,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.4362361431121826,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 1.4636783599853516,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 2.9437320232391357,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.5306460380554199,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 1.3007458448410034,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.34657702445983884,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 1.569557547569275,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.25293736457824706,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.8431988954544067,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.208502197265625,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 2.0384669303894043,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.16615729331970214,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 1.341955542564392,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.14722728729248047,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.9276875257492065,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.14779376983642578,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 1.3371649980545044,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.13147571086883544,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 1.0184792280197144,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.14180898666381836,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.5344496369361877,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.12216286659240723,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 3.0611512660980225,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.11602286100387574,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.5453760027885437,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.11221114397048951,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.4289671778678894,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.11079610586166382,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.4820654094219208,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.10537848472595215,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 87.9089,
+      "eval_generator_rouge1": 99.464,
+      "eval_generator_rouge2": 99.1899,
+      "eval_generator_rougeL": 99.0517,
+      "eval_loss": 0.009097261354327202,
+      "eval_runtime": 606.4571,
+      "eval_samples_per_second": 5.629,
+      "eval_selector_accuracy": 96.6706,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.176,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 3.0870883464813232,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.09972293376922607,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.7636368870735168,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.08893377780914306,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.4203242063522339,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.08573378920555115,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.371820867061615,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.07898830771446227,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.676804780960083,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.08897324800491332,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 1.0131059885025024,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.08602790236473083,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.4733346104621887,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.08213704824447632,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.9082027673721313,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.0822598934173584,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.3575717508792877,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0718361496925354,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.3527005612850189,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.07293112277984619,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.3359999358654022,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.07311422824859619,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.3345245122909546,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.07376066446304322,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.3286590874195099,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.06656086444854736,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.41298067569732666,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.07345538735389709,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2776452600955963,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.06613650321960449,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.27869823575019836,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.06492968797683715,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 94.743,
+      "eval_generator_rouge1": 99.8867,
+      "eval_generator_rouge2": 99.7804,
+      "eval_generator_rougeL": 99.6736,
+      "eval_loss": 0.006523935589939356,
+      "eval_runtime": 602.2662,
+      "eval_samples_per_second": 5.669,
+      "eval_selector_accuracy": 97.3715,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.178,
+      "step": 628
+    },
+    {
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.5264983177185059,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.06454238891601563,
+      "step": 640
+    },
+    {
+      "epoch": 2.102114080574392,
+      "grad_norm": 0.7168541550636292,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.06528459787368775,
+      "step": 660
+    },
+    {
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.4798216223716736,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.06159168481826782,
+      "step": 680
+    },
+    {
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.5430735349655151,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.06343585252761841,
+      "step": 700
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.2429123818874359,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.06073571443557739,
+      "step": 720
+    },
+    {
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.2920048236846924,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.05954910516738891,
+      "step": 740
+    },
+    {
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.19888697564601898,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.05689072012901306,
+      "step": 760
+    },
+    {
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.3021155893802643,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0560643196105957,
+      "step": 780
+    },
+    {
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.32512202858924866,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.0600568413734436,
+      "step": 800
+    },
+    {
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.53708815574646,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.05846037864685059,
+      "step": 820
+    },
+    {
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.3181110620498657,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.05601181983947754,
+      "step": 840
+    },
+    {
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.22879371047019958,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.05564823150634766,
+      "step": 860
+    },
+    {
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.24975994229316711,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.05476831197738648,
+      "step": 880
+    },
+    {
+      "epoch": 2.8679696848823295,
+      "grad_norm": 0.3306073546409607,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.05817422866821289,
+      "step": 900
+    },
+    {
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.1665683090686798,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.05767209529876709,
+      "step": 920
+    },
+    {
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.3881584703922272,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.05669119954109192,
+      "step": 940
+    },
+    {
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.0935,
+      "eval_generator_rouge1": 99.8705,
+      "eval_generator_rouge2": 99.7886,
+      "eval_generator_rougeL": 99.6664,
+      "eval_loss": 0.005655109416693449,
+      "eval_runtime": 604.2347,
+      "eval_samples_per_second": 5.65,
+      "eval_selector_accuracy": 98.4813,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.177,
+      "step": 942
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.524157743353037e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fdd_srkl/checkpoint-942/training_args.bin b/qwen3/fdd_srkl/checkpoint-942/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..063309915a275e078e6f834ef9c384b1cb1c965e
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb0e76106b10de1344a3493c5f04dde38b30a16fde01bc76445c98b578802418
+size 7185
diff --git a/qwen3/fdd_srkl/checkpoint-942/zero_to_fp32.py b/qwen3/fdd_srkl/checkpoint-942/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fdd_srkl/checkpoint-942/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fdd_srkl/eval_results.json b/qwen3/fdd_srkl/eval_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..94f1db0e855c4fa546d08c0896f2491c423162d0
--- /dev/null
+++ b/qwen3/fdd_srkl/eval_results.json
@@ -0,0 +1,14 @@
+{
+    "epoch": 5.0,
+    "eval_generator_count": 1712.0,
+    "eval_generator_exact_match": 96.6121,
+    "eval_generator_rouge1": 99.8663,
+    "eval_generator_rouge2": 99.793,
+    "eval_generator_rougeL": 99.724,
+    "eval_loss": 0.0041342261247336864,
+    "eval_runtime": 595.4628,
+    "eval_samples_per_second": 5.733,
+    "eval_selector_accuracy": 98.7734,
+    "eval_selector_count": 1712.0,
+    "eval_steps_per_second": 0.18
+}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/tokenizer.json b/qwen3/fdd_srkl/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fdd_srkl/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fdd_srkl/tokenizer_config.json b/qwen3/fdd_srkl/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fdd_srkl/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fdd_srkl/train_results.json b/qwen3/fdd_srkl/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..792f189b13580d3b71c7989d0ce82bb67cd1450a
--- /dev/null
+++ b/qwen3/fdd_srkl/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 5.0,
+    "total_flos": 5.871444840310374e+16,
+    "train_loss": 0.10070288321774476,
+    "train_runtime": 8259.7035,
+    "train_samples_per_second": 6.07,
+    "train_steps_per_second": 0.19
+}
\ No newline at end of file
diff --git a/qwen3/fdd_srkl/trainer_state.json b/qwen3/fdd_srkl/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..040649aabed1d023e0407c72f4adff308f6b9c67
--- /dev/null
+++ b/qwen3/fdd_srkl/trainer_state.json
@@ -0,0 +1,664 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 1570,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.4362361431121826,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 1.4636783599853516,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 2.9437320232391357,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.5306460380554199,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 1.3007458448410034,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.34657702445983884,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 1.569557547569275,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.25293736457824706,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.8431988954544067,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.208502197265625,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 2.0384669303894043,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.16615729331970214,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 1.341955542564392,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.14722728729248047,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.9276875257492065,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.14779376983642578,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 1.3371649980545044,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.13147571086883544,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 1.0184792280197144,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.14180898666381836,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.5344496369361877,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.12216286659240723,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 3.0611512660980225,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.11602286100387574,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.5453760027885437,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.11221114397048951,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.4289671778678894,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.11079610586166382,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.4820654094219208,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.10537848472595215,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 87.9089,
+      "eval_generator_rouge1": 99.464,
+      "eval_generator_rouge2": 99.1899,
+      "eval_generator_rougeL": 99.0517,
+      "eval_loss": 0.009097261354327202,
+      "eval_runtime": 606.4571,
+      "eval_samples_per_second": 5.629,
+      "eval_selector_accuracy": 96.6706,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.176,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 3.0870883464813232,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.09972293376922607,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.7636368870735168,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.08893377780914306,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.4203242063522339,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.08573378920555115,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.371820867061615,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.07898830771446227,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.676804780960083,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.08897324800491332,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 1.0131059885025024,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.08602790236473083,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.4733346104621887,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.08213704824447632,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.9082027673721313,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.0822598934173584,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.3575717508792877,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0718361496925354,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.3527005612850189,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.07293112277984619,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.3359999358654022,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.07311422824859619,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.3345245122909546,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.07376066446304322,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.3286590874195099,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.06656086444854736,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.41298067569732666,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.07345538735389709,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2776452600955963,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.06613650321960449,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.27869823575019836,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.06492968797683715,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 94.743,
+      "eval_generator_rouge1": 99.8867,
+      "eval_generator_rouge2": 99.7804,
+      "eval_generator_rougeL": 99.6736,
+      "eval_loss": 0.006523935589939356,
+      "eval_runtime": 602.2662,
+      "eval_samples_per_second": 5.669,
+      "eval_selector_accuracy": 97.3715,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.178,
+      "step": 628
+    },
+    {
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.5264983177185059,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.06454238891601563,
+      "step": 640
+    },
+    {
+      "epoch": 2.102114080574392,
+      "grad_norm": 0.7168541550636292,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.06528459787368775,
+      "step": 660
+    },
+    {
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.4798216223716736,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.06159168481826782,
+      "step": 680
+    },
+    {
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.5430735349655151,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.06343585252761841,
+      "step": 700
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.2429123818874359,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.06073571443557739,
+      "step": 720
+    },
+    {
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.2920048236846924,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.05954910516738891,
+      "step": 740
+    },
+    {
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.19888697564601898,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.05689072012901306,
+      "step": 760
+    },
+    {
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.3021155893802643,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0560643196105957,
+      "step": 780
+    },
+    {
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.32512202858924866,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.0600568413734436,
+      "step": 800
+    },
+    {
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.53708815574646,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.05846037864685059,
+      "step": 820
+    },
+    {
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.3181110620498657,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.05601181983947754,
+      "step": 840
+    },
+    {
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.22879371047019958,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.05564823150634766,
+      "step": 860
+    },
+    {
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.24975994229316711,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.05476831197738648,
+      "step": 880
+    },
+    {
+      "epoch": 2.8679696848823295,
+      "grad_norm": 0.3306073546409607,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.05817422866821289,
+      "step": 900
+    },
+    {
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.1665683090686798,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.05767209529876709,
+      "step": 920
+    },
+    {
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.3881584703922272,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.05669119954109192,
+      "step": 940
+    },
+    {
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.0935,
+      "eval_generator_rouge1": 99.8705,
+      "eval_generator_rouge2": 99.7886,
+      "eval_generator_rougeL": 99.6664,
+      "eval_loss": 0.005655109416693449,
+      "eval_runtime": 604.2347,
+      "eval_samples_per_second": 5.65,
+      "eval_selector_accuracy": 98.4813,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.177,
+      "step": 942
+    },
+    {
+      "epoch": 3.057439170323095,
+      "grad_norm": 0.2090013325214386,
+      "learning_rate": 3.3007844105971336e-05,
+      "loss": 0.05627270340919495,
+      "step": 960
+    },
+    {
+      "epoch": 3.12126047068209,
+      "grad_norm": 0.2156447023153305,
+      "learning_rate": 3.114292593987149e-05,
+      "loss": 0.05334089994430542,
+      "step": 980
+    },
+    {
+      "epoch": 3.185081771041085,
+      "grad_norm": 0.1974238008260727,
+      "learning_rate": 2.9308285746383408e-05,
+      "loss": 0.051919639110565186,
+      "step": 1000
+    },
+    {
+      "epoch": 3.24890307140008,
+      "grad_norm": 0.18332256376743317,
+      "learning_rate": 2.7506861536187867e-05,
+      "loss": 0.05656099319458008,
+      "step": 1020
+    },
+    {
+      "epoch": 3.3127243717590744,
+      "grad_norm": 0.7237991094589233,
+      "learning_rate": 2.5741538127566396e-05,
+      "loss": 0.05259966850280762,
+      "step": 1040
+    },
+    {
+      "epoch": 3.3765456721180693,
+      "grad_norm": 0.2267737239599228,
+      "learning_rate": 2.4015142526625115e-05,
+      "loss": 0.05074975490570068,
+      "step": 1060
+    },
+    {
+      "epoch": 3.4403669724770642,
+      "grad_norm": 0.20692221820354462,
+      "learning_rate": 2.233043940009967e-05,
+      "loss": 0.054996371269226074,
+      "step": 1080
+    },
+    {
+      "epoch": 3.504188272836059,
+      "grad_norm": 0.15545181930065155,
+      "learning_rate": 2.0690126647990974e-05,
+      "loss": 0.05093409419059754,
+      "step": 1100
+    },
+    {
+      "epoch": 3.568009573195054,
+      "grad_norm": 0.1771135926246643,
+      "learning_rate": 1.9096831083121828e-05,
+      "loss": 0.05136837959289551,
+      "step": 1120
+    },
+    {
+      "epoch": 3.6318308735540485,
+      "grad_norm": 0.24874085187911987,
+      "learning_rate": 1.7553104224533324e-05,
+      "loss": 0.05098183155059814,
+      "step": 1140
+    },
+    {
+      "epoch": 3.6956521739130435,
+      "grad_norm": 0.17957162857055664,
+      "learning_rate": 1.606141821145735e-05,
+      "loss": 0.050333261489868164,
+      "step": 1160
+    },
+    {
+      "epoch": 3.7594734742720384,
+      "grad_norm": 0.1954505741596222,
+      "learning_rate": 1.4624161844408763e-05,
+      "loss": 0.05178115367889404,
+      "step": 1180
+    },
+    {
+      "epoch": 3.823294774631033,
+      "grad_norm": 0.19122923910617828,
+      "learning_rate": 1.324363675973698e-05,
+      "loss": 0.050999945402145384,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8871160749900278,
+      "grad_norm": 0.26517826318740845,
+      "learning_rate": 1.1922053743763052e-05,
+      "loss": 0.0516182005405426,
+      "step": 1220
+    },
+    {
+      "epoch": 3.9509373753490227,
+      "grad_norm": 0.1767585426568985,
+      "learning_rate": 1.0661529192404973e-05,
+      "loss": 0.04989046454429626,
+      "step": 1240
+    },
+    {
+      "epoch": 4.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.3201,
+      "eval_generator_rouge1": 99.9029,
+      "eval_generator_rouge2": 99.8325,
+      "eval_generator_rougeL": 99.7566,
+      "eval_loss": 0.0041970605961978436,
+      "eval_runtime": 743.2973,
+      "eval_samples_per_second": 4.593,
+      "eval_selector_accuracy": 98.5397,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.144,
+      "step": 1256
+    },
+    {
+      "epoch": 4.0127642600717985,
+      "grad_norm": 0.17936867475509644,
+      "learning_rate": 9.464081721960592e-06,
+      "loss": 0.05013519525527954,
+      "step": 1260
+    },
+    {
+      "epoch": 4.076585560430794,
+      "grad_norm": 0.323055237531662,
+      "learning_rate": 8.331628936475818e-06,
+      "loss": 0.04865485727787018,
+      "step": 1280
+    },
+    {
+      "epoch": 4.140406860789788,
+      "grad_norm": 0.16135305166244507,
+      "learning_rate": 7.2659843568748385e-06,
+      "loss": 0.04849388599395752,
+      "step": 1300
+    },
+    {
+      "epoch": 4.204228161148784,
+      "grad_norm": 0.17001892626285553,
+      "learning_rate": 6.268854516770008e-06,
+      "loss": 0.04871906638145447,
+      "step": 1320
+    },
+    {
+      "epoch": 4.268049461507778,
+      "grad_norm": 0.18477433919906616,
+      "learning_rate": 5.341836229602331e-06,
+      "loss": 0.048744958639144895,
+      "step": 1340
+    },
+    {
+      "epoch": 4.331870761866773,
+      "grad_norm": 0.1783062219619751,
+      "learning_rate": 4.4864140314888216e-06,
+      "loss": 0.04793646335601807,
+      "step": 1360
+    },
+    {
+      "epoch": 4.395692062225768,
+      "grad_norm": 0.17276793718338013,
+      "learning_rate": 3.703957803871932e-06,
+      "loss": 0.048929840326309204,
+      "step": 1380
+    },
+    {
+      "epoch": 4.4595133625847625,
+      "grad_norm": 0.1627519130706787,
+      "learning_rate": 2.9957205797780495e-06,
+      "loss": 0.04951263666152954,
+      "step": 1400
+    },
+    {
+      "epoch": 4.523334662943758,
+      "grad_norm": 0.28899943828582764,
+      "learning_rate": 2.362836537198199e-06,
+      "loss": 0.05062735080718994,
+      "step": 1420
+    },
+    {
+      "epoch": 4.587155963302752,
+      "grad_norm": 0.17383745312690735,
+      "learning_rate": 1.8063191828043703e-06,
+      "loss": 0.049233487248420714,
+      "step": 1440
+    },
+    {
+      "epoch": 4.650977263661747,
+      "grad_norm": 0.15494407713413239,
+      "learning_rate": 1.3270597289100437e-06,
+      "loss": 0.04810199737548828,
+      "step": 1460
+    },
+    {
+      "epoch": 4.714798564020742,
+      "grad_norm": 0.14835470914840698,
+      "learning_rate": 9.258256662741205e-07,
+      "loss": 0.04752459526062012,
+      "step": 1480
+    },
+    {
+      "epoch": 4.778619864379737,
+      "grad_norm": 0.17123174667358398,
+      "learning_rate": 6.032595350337341e-07,
+      "loss": 0.04880803823471069,
+      "step": 1500
+    },
+    {
+      "epoch": 4.842441164738732,
+      "grad_norm": 0.15825852751731873,
+      "learning_rate": 3.5987789573421356e-07,
+      "loss": 0.05428855419158936,
+      "step": 1520
+    },
+    {
+      "epoch": 4.9062624650977265,
+      "grad_norm": 1.0737415552139282,
+      "learning_rate": 1.9607050210401192e-07,
+      "loss": 0.0523545503616333,
+      "step": 1540
+    },
+    {
+      "epoch": 4.970083765456721,
+      "grad_norm": 0.18025070428848267,
+      "learning_rate": 1.1209967689925832e-07,
+      "loss": 0.049212086200714114,
+      "step": 1560
+    },
+    {
+      "epoch": 5.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.7874,
+      "eval_generator_rouge1": 99.9018,
+      "eval_generator_rouge2": 99.8253,
+      "eval_generator_rougeL": 99.7687,
+      "eval_loss": 0.0041342261247336864,
+      "eval_runtime": 612.0487,
+      "eval_samples_per_second": 5.578,
+      "eval_selector_accuracy": 98.8318,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 1570
+    },
+    {
+      "epoch": 5.0,
+      "step": 1570,
+      "total_flos": 5.871444840310374e+16,
+      "train_loss": 0.10070288321774476,
+      "train_runtime": 8259.7035,
+      "train_samples_per_second": 6.07,
+      "train_steps_per_second": 0.19
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.871444840310374e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fdd_srkl/training_args.bin b/qwen3/fdd_srkl/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..063309915a275e078e6f834ef9c384b1cb1c965e
--- /dev/null
+++ b/qwen3/fdd_srkl/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb0e76106b10de1344a3493c5f04dde38b30a16fde01bc76445c98b578802418
+size 7185
diff --git a/qwen3/fkl/README.md b/qwen3/fkl/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..8a3c8b9ced347e576e6c335a93dbb2a338e1cef0
--- /dev/null
+++ b/qwen3/fkl/README.md
@@ -0,0 +1,79 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen3-0.6B
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: fkl
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# fkl
+
+This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0035
+- Selector Count: 1712.0
+- Generator Count: 1712.0
+- Selector Accuracy: 98.6565
+- Generator Exact Match: 96.9042
+- Generator Rouge1: 99.9327
+- Generator Rouge2: 99.876
+- Generator Rougel: 99.7659
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 2
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_min_lr
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:|
+| 0.0137        | 1.0   | 314  | 0.0064          | 1712.0         | 1712.0          | 95.3855           | 91.764                | 99.6596          | 99.4475          | 99.3289          |
+| 0.0038        | 2.0   | 628  | 0.0043          | 1712.0         | 1712.0          | 97.5467           | 93.6916               | 99.7589          | 99.5845          | 99.5171          |
+| 0.0013        | 3.0   | 942  | 0.0034          | 1712.0         | 1712.0          | 98.8318           | 95.5023               | 99.9079          | 99.8141          | 99.6685          |
+| 0.0010        | 4.0   | 1256 | 0.0035          | 1712.0         | 1712.0          | 98.5397           | 96.3785               | 99.9022          | 99.8287          | 99.7265          |
+| 0.0008        | 5.0   | 1570 | 0.0035          | 1712.0         | 1712.0          | 98.6565           | 95.9696               | 99.9236          | 99.8518          | 99.7241          |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.8.0
+- Pytorch 2.9.1+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2
\ No newline at end of file
diff --git a/qwen3/fkl/adapter_config.json b/qwen3/fkl/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ba50532007224c0205252929f5d7a29f112a16
--- /dev/null
+++ b/qwen3/fkl/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fkl/adapter_model.safetensors b/qwen3/fkl/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0c39526dde863f4798725640906a8bf2419dad19
--- /dev/null
+++ b/qwen3/fkl/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98711a02309781c3eb7ebf1ba97bb14165c484a9f68c8a79b96126871c1deb7b
+size 40422560
diff --git a/qwen3/fkl/all_results.json b/qwen3/fkl/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ede5a31471eb74d465c45e0c2ad404d30c60eee
--- /dev/null
+++ b/qwen3/fkl/all_results.json
@@ -0,0 +1,19 @@
+{
+    "epoch": 5.0,
+    "eval_generator_count": 1712.0,
+    "eval_generator_exact_match": 96.9042,
+    "eval_generator_rouge1": 99.9327,
+    "eval_generator_rouge2": 99.876,
+    "eval_generator_rougeL": 99.7659,
+    "eval_loss": 0.0034750187769532204,
+    "eval_runtime": 599.1469,
+    "eval_samples_per_second": 5.698,
+    "eval_selector_accuracy": 98.6565,
+    "eval_selector_count": 1712.0,
+    "eval_steps_per_second": 0.179,
+    "total_flos": 5.871444840310374e+16,
+    "train_loss": 0.015215813707275564,
+    "train_runtime": 8273.9277,
+    "train_samples_per_second": 6.059,
+    "train_steps_per_second": 0.19
+}
\ No newline at end of file
diff --git a/qwen3/fkl/chat_template.jinja b/qwen3/fkl/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fkl/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1256/README.md b/qwen3/fkl/checkpoint-1256/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1256/adapter_config.json b/qwen3/fkl/checkpoint-1256/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ba50532007224c0205252929f5d7a29f112a16
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1256/adapter_model.safetensors b/qwen3/fkl/checkpoint-1256/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a6f547014dcab092a23ed6f82272ec4dc2f20069
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e4398ae452bf5034b81083cf6ed15f7c6d84a5e303755bf0f5ae34a1ec97b00
+size 40422560
diff --git a/qwen3/fkl/checkpoint-1256/chat_template.jinja b/qwen3/fkl/checkpoint-1256/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c04eecbaede837bf2b16440e4bf1bf1e905a3a00
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52aa08d80734b3c38a49791c3dde8efc4de7b1f81886522eb63a26b6bb641a2e
+size 121135557
diff --git a/qwen3/fkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e3127d4318b7141a149efa60f89a33e50603248
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d454dd6fc29a810783c042d6795f83e8033a109e6760404c5819f81305700fd2
+size 121135749
diff --git a/qwen3/fkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/fkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecef40af22e40db0dc1a1c246a40e684d69480f2
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28c024879e822d33797b41094d715daa03e8e4779d7522fc971c9e5588ee4ef8
+size 351841765
diff --git a/qwen3/fkl/checkpoint-1256/latest b/qwen3/fkl/checkpoint-1256/latest
new file mode 100644
index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/latest
@@ -0,0 +1 @@
+global_step1256
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1256/rng_state_0.pth b/qwen3/fkl/checkpoint-1256/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..dea577c1e629718b2015b832dea5e1e88e418ebb
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6732df283ebc0c27eda5b5a1de99db7991fdca27fc172a6d195a532b85c238d
+size 14917
diff --git a/qwen3/fkl/checkpoint-1256/rng_state_1.pth b/qwen3/fkl/checkpoint-1256/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8cc175733fb724879ba7ee961de4b071c7b7cf05
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57a41bfc1a372c355188ea4b92985f0a3fde224f5db38b61b66d27892f62a72a
+size 14917
diff --git a/qwen3/fkl/checkpoint-1256/scheduler.pt b/qwen3/fkl/checkpoint-1256/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23
+size 1465
diff --git a/qwen3/fkl/checkpoint-1256/tokenizer.json b/qwen3/fkl/checkpoint-1256/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fkl/checkpoint-1256/tokenizer_config.json b/qwen3/fkl/checkpoint-1256/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fkl/checkpoint-1256/trainer_state.json b/qwen3/fkl/checkpoint-1256/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..817c2bcad34e8874c8a6449f3f005bc32e8f4b38
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/trainer_state.json
@@ -0,0 +1,528 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 1256,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.7414703369140625,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.5651369094848633,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 0.7481251358985901,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.10692100524902344,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 0.8983101844787598,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.0871108591556549,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 0.3072285056114197,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.03171964287757874,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.5620023608207703,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.038917267322540285,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 0.34996840357780457,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.02393315136432648,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.285879522562027,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.0288031667470932,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 0.28871023654937744,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.020403537154197692,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 0.3916277289390564,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.017323681712150575,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.15098990499973297,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.022147342562675476,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.4012651741504669,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01552567332983017,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 0.48113393783569336,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.012960800528526306,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.6242109537124634,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.012793044745922088,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.29736602306365967,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.014466887712478638,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.320481538772583,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013739512860774994,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 91.764,
+      "eval_generator_rouge1": 99.6596,
+      "eval_generator_rouge2": 99.4475,
+      "eval_generator_rougeL": 99.3289,
+      "eval_loss": 0.006375482305884361,
+      "eval_runtime": 611.1452,
+      "eval_samples_per_second": 5.586,
+      "eval_selector_accuracy": 95.3855,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 1.464452862739563,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014014726877212525,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.08158843219280243,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.006687680631875992,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.12018018215894699,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.0065675824880599976,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.20543402433395386,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.004217476397752762,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.15771833062171936,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.010484220832586289,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.1818164736032486,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.009176531434059143,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.043983329087495804,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.004671639204025269,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.2847776710987091,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.003687678650021553,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.4654720425605774,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0022115066647529604,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.057350289076566696,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.006766787171363831,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.038868289440870285,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.007653500884771347,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.05846746638417244,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.00647004097700119,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.16134238243103027,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.0035313751548528673,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 2.057159423828125,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.008149225264787674,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 1.8178443908691406,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.00631907656788826,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.023604823276400566,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.003845648840069771,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 93.6916,
+      "eval_generator_rouge1": 99.7589,
+      "eval_generator_rouge2": 99.5845,
+      "eval_generator_rougeL": 99.5171,
+      "eval_loss": 0.004298548214137554,
+      "eval_runtime": 611.3037,
+      "eval_samples_per_second": 5.585,
+      "eval_selector_accuracy": 97.5467,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 628
+    },
+    {
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.07722756266593933,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.0037871412932872772,
+      "step": 640
+    },
+    {
+      "epoch": 2.102114080574392,
+      "grad_norm": 0.4329485297203064,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.006162230670452118,
+      "step": 660
+    },
+    {
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.15261922776699066,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.0018202636390924453,
+      "step": 680
+    },
+    {
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.031139470636844635,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.0023104183375835417,
+      "step": 700
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.018401559442281723,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.0016305617988109588,
+      "step": 720
+    },
+    {
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.03898131102323532,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.0027825018391013146,
+      "step": 740
+    },
+    {
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.008162672631442547,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.0016868755221366883,
+      "step": 760
+    },
+    {
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.05951504036784172,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0010992977768182755,
+      "step": 780
+    },
+    {
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.08524229377508163,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.0025967489928007126,
+      "step": 800
+    },
+    {
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.039385274052619934,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.0020601760596036913,
+      "step": 820
+    },
+    {
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.14205299317836761,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.002689024433493614,
+      "step": 840
+    },
+    {
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.07378510385751724,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.0022501811385154725,
+      "step": 860
+    },
+    {
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.16569221019744873,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.0014809093438088894,
+      "step": 880
+    },
+    {
+      "epoch": 2.8679696848823295,
+      "grad_norm": 0.12665939331054688,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.0022083733230829237,
+      "step": 900
+    },
+    {
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.005265355110168457,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.002069174498319626,
+      "step": 920
+    },
+    {
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.10703321546316147,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.0012829974293708801,
+      "step": 940
+    },
+    {
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.5023,
+      "eval_generator_rouge1": 99.9079,
+      "eval_generator_rouge2": 99.8141,
+      "eval_generator_rougeL": 99.6685,
+      "eval_loss": 0.00343098072335124,
+      "eval_runtime": 620.0123,
+      "eval_samples_per_second": 5.506,
+      "eval_selector_accuracy": 98.8318,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.173,
+      "step": 942
+    },
+    {
+      "epoch": 3.057439170323095,
+      "grad_norm": 0.023057373240590096,
+      "learning_rate": 3.3007844105971336e-05,
+      "loss": 0.0013455232605338097,
+      "step": 960
+    },
+    {
+      "epoch": 3.12126047068209,
+      "grad_norm": 0.006213244516402483,
+      "learning_rate": 3.114292593987149e-05,
+      "loss": 0.001208091713488102,
+      "step": 980
+    },
+    {
+      "epoch": 3.185081771041085,
+      "grad_norm": 0.0030793482437729836,
+      "learning_rate": 2.9308285746383408e-05,
+      "loss": 0.0011879941448569298,
+      "step": 1000
+    },
+    {
+      "epoch": 3.24890307140008,
+      "grad_norm": 0.025737427175045013,
+      "learning_rate": 2.7506861536187867e-05,
+      "loss": 0.0008374839089810848,
+      "step": 1020
+    },
+    {
+      "epoch": 3.3127243717590744,
+      "grad_norm": 0.03453245759010315,
+      "learning_rate": 2.5741538127566396e-05,
+      "loss": 0.0008559312671422958,
+      "step": 1040
+    },
+    {
+      "epoch": 3.3765456721180693,
+      "grad_norm": 0.001912926440127194,
+      "learning_rate": 2.4015142526625115e-05,
+      "loss": 0.0016484295949339867,
+      "step": 1060
+    },
+    {
+      "epoch": 3.4403669724770642,
+      "grad_norm": 0.07635863870382309,
+      "learning_rate": 2.233043940009967e-05,
+      "loss": 0.0025718800723552706,
+      "step": 1080
+    },
+    {
+      "epoch": 3.504188272836059,
+      "grad_norm": 0.015918560326099396,
+      "learning_rate": 2.0690126647990974e-05,
+      "loss": 0.0009869973175227642,
+      "step": 1100
+    },
+    {
+      "epoch": 3.568009573195054,
+      "grad_norm": 0.014826856553554535,
+      "learning_rate": 1.9096831083121828e-05,
+      "loss": 0.0014582538977265358,
+      "step": 1120
+    },
+    {
+      "epoch": 3.6318308735540485,
+      "grad_norm": 0.020788000896573067,
+      "learning_rate": 1.7553104224533324e-05,
+      "loss": 0.000823317002505064,
+      "step": 1140
+    },
+    {
+      "epoch": 3.6956521739130435,
+      "grad_norm": 0.005068339407444,
+      "learning_rate": 1.606141821145735e-05,
+      "loss": 0.001963364891707897,
+      "step": 1160
+    },
+    {
+      "epoch": 3.7594734742720384,
+      "grad_norm": 0.012145614251494408,
+      "learning_rate": 1.4624161844408763e-05,
+      "loss": 0.0021551501005887983,
+      "step": 1180
+    },
+    {
+      "epoch": 3.823294774631033,
+      "grad_norm": 0.011486915871500969,
+      "learning_rate": 1.324363675973698e-05,
+      "loss": 0.0011032622307538987,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8871160749900278,
+      "grad_norm": 0.06560127437114716,
+      "learning_rate": 1.1922053743763052e-05,
+      "loss": 0.0013195869512856006,
+      "step": 1220
+    },
+    {
+      "epoch": 3.9509373753490227,
+      "grad_norm": 0.014271366409957409,
+      "learning_rate": 1.0661529192404973e-05,
+      "loss": 0.0009616646915674209,
+      "step": 1240
+    },
+    {
+      "epoch": 4.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.3785,
+      "eval_generator_rouge1": 99.9022,
+      "eval_generator_rouge2": 99.8287,
+      "eval_generator_rougeL": 99.7265,
+      "eval_loss": 0.0034609222784638405,
+      "eval_runtime": 688.7408,
+      "eval_samples_per_second": 4.957,
+      "eval_selector_accuracy": 98.5397,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.155,
+      "step": 1256
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.697643190059008e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fkl/checkpoint-1256/training_args.bin b/qwen3/fkl/checkpoint-1256/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..62f02c6f015c2ecc24752edd527d8a455bc9b089
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca0c534c8077d1140a5d674eead223cc4f38c64dc44b4464a6525db759412db
+size 7185
diff --git a/qwen3/fkl/checkpoint-1256/zero_to_fp32.py b/qwen3/fkl/checkpoint-1256/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1256/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fkl/checkpoint-1570/README.md b/qwen3/fkl/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1570/adapter_config.json b/qwen3/fkl/checkpoint-1570/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ba50532007224c0205252929f5d7a29f112a16
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1570/adapter_model.safetensors b/qwen3/fkl/checkpoint-1570/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0c39526dde863f4798725640906a8bf2419dad19
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98711a02309781c3eb7ebf1ba97bb14165c484a9f68c8a79b96126871c1deb7b
+size 40422560
diff --git a/qwen3/fkl/checkpoint-1570/chat_template.jinja b/qwen3/fkl/checkpoint-1570/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6464f48758adce02f84754fea3297ffc8a90f97e
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ba866805e95e1887770098ee1b12f33aed2a7e52b256c3fd9f791440ae985aa
+size 121135557
diff --git a/qwen3/fkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..85f4753e7da72b99c5d32da739adc07696a304bc
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb91700eb4a198bb9ffaaef62f0a0b322d6ce4d7ef7ed64dc15d56d3cf44a5f8
+size 121135749
diff --git a/qwen3/fkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/fkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99f666d4cf7b6bee69cab1b34fb0be088de03a7c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a75997c6927ce1a9ce1c93706bd6bad2ce399b2ecd8d4c064a79ba7184fbd5b
+size 351841765
diff --git a/qwen3/fkl/checkpoint-1570/latest b/qwen3/fkl/checkpoint-1570/latest
new file mode 100644
index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/latest
@@ -0,0 +1 @@
+global_step1570
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-1570/rng_state_0.pth b/qwen3/fkl/checkpoint-1570/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6be962ad3c2ed8589eb19f94452c6e5d3e76768b
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2986f581abdabea5fc0dbf93a024a06de438bd53e28442a8349810c7308c581a
+size 14917
diff --git a/qwen3/fkl/checkpoint-1570/rng_state_1.pth b/qwen3/fkl/checkpoint-1570/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e09c2741dd53615d272fb5b7462241a5338a0cac
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0f9e8d5df06e1d9e41516db69f757cc195ade10cdd3f83760d824e1d361ea27
+size 14917
diff --git a/qwen3/fkl/checkpoint-1570/scheduler.pt b/qwen3/fkl/checkpoint-1570/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae
+size 1465
diff --git a/qwen3/fkl/checkpoint-1570/tokenizer.json b/qwen3/fkl/checkpoint-1570/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fkl/checkpoint-1570/tokenizer_config.json b/qwen3/fkl/checkpoint-1570/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fkl/checkpoint-1570/trainer_state.json b/qwen3/fkl/checkpoint-1570/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..399eaba602dbdec5959161f8c7c7cfa345c13695
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/trainer_state.json
@@ -0,0 +1,655 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 1570,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.7414703369140625,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.5651369094848633,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 0.7481251358985901,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.10692100524902344,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 0.8983101844787598,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.0871108591556549,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 0.3072285056114197,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.03171964287757874,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.5620023608207703,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.038917267322540285,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 0.34996840357780457,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.02393315136432648,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.285879522562027,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.0288031667470932,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 0.28871023654937744,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.020403537154197692,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 0.3916277289390564,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.017323681712150575,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.15098990499973297,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.022147342562675476,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.4012651741504669,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01552567332983017,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 0.48113393783569336,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.012960800528526306,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.6242109537124634,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.012793044745922088,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.29736602306365967,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.014466887712478638,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.320481538772583,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013739512860774994,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 91.764,
+      "eval_generator_rouge1": 99.6596,
+      "eval_generator_rouge2": 99.4475,
+      "eval_generator_rougeL": 99.3289,
+      "eval_loss": 0.006375482305884361,
+      "eval_runtime": 611.1452,
+      "eval_samples_per_second": 5.586,
+      "eval_selector_accuracy": 95.3855,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 1.464452862739563,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014014726877212525,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.08158843219280243,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.006687680631875992,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.12018018215894699,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.0065675824880599976,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.20543402433395386,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.004217476397752762,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.15771833062171936,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.010484220832586289,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.1818164736032486,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.009176531434059143,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.043983329087495804,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.004671639204025269,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.2847776710987091,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.003687678650021553,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.4654720425605774,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0022115066647529604,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.057350289076566696,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.006766787171363831,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.038868289440870285,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.007653500884771347,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.05846746638417244,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.00647004097700119,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.16134238243103027,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.0035313751548528673,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 2.057159423828125,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.008149225264787674,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 1.8178443908691406,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.00631907656788826,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.023604823276400566,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.003845648840069771,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 93.6916,
+      "eval_generator_rouge1": 99.7589,
+      "eval_generator_rouge2": 99.5845,
+      "eval_generator_rougeL": 99.5171,
+      "eval_loss": 0.004298548214137554,
+      "eval_runtime": 611.3037,
+      "eval_samples_per_second": 5.585,
+      "eval_selector_accuracy": 97.5467,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 628
+    },
+    {
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.07722756266593933,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.0037871412932872772,
+      "step": 640
+    },
+    {
+      "epoch": 2.102114080574392,
+      "grad_norm": 0.4329485297203064,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.006162230670452118,
+      "step": 660
+    },
+    {
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.15261922776699066,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.0018202636390924453,
+      "step": 680
+    },
+    {
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.031139470636844635,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.0023104183375835417,
+      "step": 700
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.018401559442281723,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.0016305617988109588,
+      "step": 720
+    },
+    {
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.03898131102323532,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.0027825018391013146,
+      "step": 740
+    },
+    {
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.008162672631442547,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.0016868755221366883,
+      "step": 760
+    },
+    {
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.05951504036784172,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0010992977768182755,
+      "step": 780
+    },
+    {
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.08524229377508163,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.0025967489928007126,
+      "step": 800
+    },
+    {
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.039385274052619934,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.0020601760596036913,
+      "step": 820
+    },
+    {
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.14205299317836761,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.002689024433493614,
+      "step": 840
+    },
+    {
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.07378510385751724,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.0022501811385154725,
+      "step": 860
+    },
+    {
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.16569221019744873,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.0014809093438088894,
+      "step": 880
+    },
+    {
+      "epoch": 2.8679696848823295,
+      "grad_norm": 0.12665939331054688,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.0022083733230829237,
+      "step": 900
+    },
+    {
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.005265355110168457,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.002069174498319626,
+      "step": 920
+    },
+    {
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.10703321546316147,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.0012829974293708801,
+      "step": 940
+    },
+    {
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.5023,
+      "eval_generator_rouge1": 99.9079,
+      "eval_generator_rouge2": 99.8141,
+      "eval_generator_rougeL": 99.6685,
+      "eval_loss": 0.00343098072335124,
+      "eval_runtime": 620.0123,
+      "eval_samples_per_second": 5.506,
+      "eval_selector_accuracy": 98.8318,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.173,
+      "step": 942
+    },
+    {
+      "epoch": 3.057439170323095,
+      "grad_norm": 0.023057373240590096,
+      "learning_rate": 3.3007844105971336e-05,
+      "loss": 0.0013455232605338097,
+      "step": 960
+    },
+    {
+      "epoch": 3.12126047068209,
+      "grad_norm": 0.006213244516402483,
+      "learning_rate": 3.114292593987149e-05,
+      "loss": 0.001208091713488102,
+      "step": 980
+    },
+    {
+      "epoch": 3.185081771041085,
+      "grad_norm": 0.0030793482437729836,
+      "learning_rate": 2.9308285746383408e-05,
+      "loss": 0.0011879941448569298,
+      "step": 1000
+    },
+    {
+      "epoch": 3.24890307140008,
+      "grad_norm": 0.025737427175045013,
+      "learning_rate": 2.7506861536187867e-05,
+      "loss": 0.0008374839089810848,
+      "step": 1020
+    },
+    {
+      "epoch": 3.3127243717590744,
+      "grad_norm": 0.03453245759010315,
+      "learning_rate": 2.5741538127566396e-05,
+      "loss": 0.0008559312671422958,
+      "step": 1040
+    },
+    {
+      "epoch": 3.3765456721180693,
+      "grad_norm": 0.001912926440127194,
+      "learning_rate": 2.4015142526625115e-05,
+      "loss": 0.0016484295949339867,
+      "step": 1060
+    },
+    {
+      "epoch": 3.4403669724770642,
+      "grad_norm": 0.07635863870382309,
+      "learning_rate": 2.233043940009967e-05,
+      "loss": 0.0025718800723552706,
+      "step": 1080
+    },
+    {
+      "epoch": 3.504188272836059,
+      "grad_norm": 0.015918560326099396,
+      "learning_rate": 2.0690126647990974e-05,
+      "loss": 0.0009869973175227642,
+      "step": 1100
+    },
+    {
+      "epoch": 3.568009573195054,
+      "grad_norm": 0.014826856553554535,
+      "learning_rate": 1.9096831083121828e-05,
+      "loss": 0.0014582538977265358,
+      "step": 1120
+    },
+    {
+      "epoch": 3.6318308735540485,
+      "grad_norm": 0.020788000896573067,
+      "learning_rate": 1.7553104224533324e-05,
+      "loss": 0.000823317002505064,
+      "step": 1140
+    },
+    {
+      "epoch": 3.6956521739130435,
+      "grad_norm": 0.005068339407444,
+      "learning_rate": 1.606141821145735e-05,
+      "loss": 0.001963364891707897,
+      "step": 1160
+    },
+    {
+      "epoch": 3.7594734742720384,
+      "grad_norm": 0.012145614251494408,
+      "learning_rate": 1.4624161844408763e-05,
+      "loss": 0.0021551501005887983,
+      "step": 1180
+    },
+    {
+      "epoch": 3.823294774631033,
+      "grad_norm": 0.011486915871500969,
+      "learning_rate": 1.324363675973698e-05,
+      "loss": 0.0011032622307538987,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8871160749900278,
+      "grad_norm": 0.06560127437114716,
+      "learning_rate": 1.1922053743763052e-05,
+      "loss": 0.0013195869512856006,
+      "step": 1220
+    },
+    {
+      "epoch": 3.9509373753490227,
+      "grad_norm": 0.014271366409957409,
+      "learning_rate": 1.0661529192404973e-05,
+      "loss": 0.0009616646915674209,
+      "step": 1240
+    },
+    {
+      "epoch": 4.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.3785,
+      "eval_generator_rouge1": 99.9022,
+      "eval_generator_rouge2": 99.8287,
+      "eval_generator_rougeL": 99.7265,
+      "eval_loss": 0.0034609222784638405,
+      "eval_runtime": 688.7408,
+      "eval_samples_per_second": 4.957,
+      "eval_selector_accuracy": 98.5397,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.155,
+      "step": 1256
+    },
+    {
+      "epoch": 4.0127642600717985,
+      "grad_norm": 0.026660269126296043,
+      "learning_rate": 9.464081721960592e-06,
+      "loss": 0.0014410967007279396,
+      "step": 1260
+    },
+    {
+      "epoch": 4.076585560430794,
+      "grad_norm": 0.055254511535167694,
+      "learning_rate": 8.331628936475818e-06,
+      "loss": 0.0007118118926882743,
+      "step": 1280
+    },
+    {
+      "epoch": 4.140406860789788,
+      "grad_norm": 0.0038272945675998926,
+      "learning_rate": 7.2659843568748385e-06,
+      "loss": 0.0010564759373664855,
+      "step": 1300
+    },
+    {
+      "epoch": 4.204228161148784,
+      "grad_norm": 0.009401796385645866,
+      "learning_rate": 6.268854516770008e-06,
+      "loss": 0.0007780813612043858,
+      "step": 1320
+    },
+    {
+      "epoch": 4.268049461507778,
+      "grad_norm": 0.03749614953994751,
+      "learning_rate": 5.341836229602331e-06,
+      "loss": 0.0010510680265724659,
+      "step": 1340
+    },
+    {
+      "epoch": 4.331870761866773,
+      "grad_norm": 0.008090543560683727,
+      "learning_rate": 4.4864140314888216e-06,
+      "loss": 0.0007310851942747831,
+      "step": 1360
+    },
+    {
+      "epoch": 4.395692062225768,
+      "grad_norm": 0.003737018443644047,
+      "learning_rate": 3.703957803871932e-06,
+      "loss": 0.000812545046210289,
+      "step": 1380
+    },
+    {
+      "epoch": 4.4595133625847625,
+      "grad_norm": 0.03762929514050484,
+      "learning_rate": 2.9957205797780495e-06,
+      "loss": 0.0015857372432947158,
+      "step": 1400
+    },
+    {
+      "epoch": 4.523334662943758,
+      "grad_norm": 0.04096720740199089,
+      "learning_rate": 2.362836537198199e-06,
+      "loss": 0.001064287591725588,
+      "step": 1420
+    },
+    {
+      "epoch": 4.587155963302752,
+      "grad_norm": 0.015219061635434628,
+      "learning_rate": 1.8063191828043703e-06,
+      "loss": 0.0007133203092962503,
+      "step": 1440
+    },
+    {
+      "epoch": 4.650977263661747,
+      "grad_norm": 0.019106416031718254,
+      "learning_rate": 1.3270597289100437e-06,
+      "loss": 0.0007402435876429081,
+      "step": 1460
+    },
+    {
+      "epoch": 4.714798564020742,
+      "grad_norm": 0.03157809376716614,
+      "learning_rate": 9.258256662741205e-07,
+      "loss": 0.0008887875825166702,
+      "step": 1480
+    },
+    {
+      "epoch": 4.778619864379737,
+      "grad_norm": 0.008441638201475143,
+      "learning_rate": 6.032595350337341e-07,
+      "loss": 0.0012488246895372868,
+      "step": 1500
+    },
+    {
+      "epoch": 4.842441164738732,
+      "grad_norm": 0.027420049533247948,
+      "learning_rate": 3.5987789573421356e-07,
+      "loss": 0.004218760132789612,
+      "step": 1520
+    },
+    {
+      "epoch": 4.9062624650977265,
+      "grad_norm": 1.1041139364242554,
+      "learning_rate": 1.9607050210401192e-07,
+      "loss": 0.001393320318311453,
+      "step": 1540
+    },
+    {
+      "epoch": 4.970083765456721,
+      "grad_norm": 0.025242790579795837,
+      "learning_rate": 1.1209967689925832e-07,
+      "loss": 0.0008278143592178822,
+      "step": 1560
+    },
+    {
+      "epoch": 5.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.9696,
+      "eval_generator_rouge1": 99.9236,
+      "eval_generator_rouge2": 99.8518,
+      "eval_generator_rougeL": 99.7241,
+      "eval_loss": 0.0034750187769532204,
+      "eval_runtime": 712.376,
+      "eval_samples_per_second": 4.792,
+      "eval_selector_accuracy": 98.6565,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.15,
+      "step": 1570
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.871444840310374e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fkl/checkpoint-1570/training_args.bin b/qwen3/fkl/checkpoint-1570/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..62f02c6f015c2ecc24752edd527d8a455bc9b089
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca0c534c8077d1140a5d674eead223cc4f38c64dc44b4464a6525db759412db
+size 7185
diff --git a/qwen3/fkl/checkpoint-1570/zero_to_fp32.py b/qwen3/fkl/checkpoint-1570/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fkl/checkpoint-1570/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fkl/checkpoint-314/README.md b/qwen3/fkl/checkpoint-314/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-314/adapter_config.json b/qwen3/fkl/checkpoint-314/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ba50532007224c0205252929f5d7a29f112a16
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-314/adapter_model.safetensors b/qwen3/fkl/checkpoint-314/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4e43deb37e0fb332afb4224369c3260a071cd9f3
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8f2321a6b9d2057732396922961025c29d69425947563501caa9a9977b59f19
+size 40422560
diff --git a/qwen3/fkl/checkpoint-314/chat_template.jinja b/qwen3/fkl/checkpoint-314/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c841c838a62de20a82b927eefeac7a2b1fcf3867
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0813cf9871e6761d3429d56ae4caaf904160dd61c7e187bb13a32d6b921309fc
+size 121135557
diff --git a/qwen3/fkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83b9182c3c5da8dfa00dcf432f75d4fdb4d8f43e
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23a690bcbd6bf3919aa9bd4ec12ddbb13adad3df7289debab982f8ce2b907e57
+size 121135749
diff --git a/qwen3/fkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/fkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1668bbe2311f0beb7b15764096affb1e670e5da3
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b073c1852ea281a36dbd2df0172777de3673a732d71d923998f40813a46eb90
+size 351841765
diff --git a/qwen3/fkl/checkpoint-314/latest b/qwen3/fkl/checkpoint-314/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/latest
@@ -0,0 +1 @@
+global_step314
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-314/rng_state_0.pth b/qwen3/fkl/checkpoint-314/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7be138a53bc9a817c59a9a3c9c6005910a2d20df
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acb9654f95c771075bf1303be82ba6db5a454c724a18a252513726da1f7261c0
+size 14917
diff --git a/qwen3/fkl/checkpoint-314/rng_state_1.pth b/qwen3/fkl/checkpoint-314/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..bc137998fed32b09e96b59b1cea43c3eaae929d5
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f9e518dc5b9efe3bc3c860032ec682318c40107ab46c5d5c5e49287a1149d61
+size 14917
diff --git a/qwen3/fkl/checkpoint-314/scheduler.pt b/qwen3/fkl/checkpoint-314/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149
+size 1465
diff --git a/qwen3/fkl/checkpoint-314/tokenizer.json b/qwen3/fkl/checkpoint-314/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fkl/checkpoint-314/tokenizer_config.json b/qwen3/fkl/checkpoint-314/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fkl/checkpoint-314/trainer_state.json b/qwen3/fkl/checkpoint-314/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..831a5793859d4bfc491faeac2c16936bd7b43b98
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/trainer_state.json
@@ -0,0 +1,154 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 314,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.7414703369140625,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.5651369094848633,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 0.7481251358985901,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.10692100524902344,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 0.8983101844787598,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.0871108591556549,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 0.3072285056114197,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.03171964287757874,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.5620023608207703,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.038917267322540285,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 0.34996840357780457,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.02393315136432648,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.285879522562027,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.0288031667470932,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 0.28871023654937744,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.020403537154197692,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 0.3916277289390564,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.017323681712150575,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.15098990499973297,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.022147342562675476,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.4012651741504669,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01552567332983017,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 0.48113393783569336,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.012960800528526306,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.6242109537124634,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.012793044745922088,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.29736602306365967,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.014466887712478638,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.320481538772583,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013739512860774994,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 91.764,
+      "eval_generator_rouge1": 99.6596,
+      "eval_generator_rouge2": 99.4475,
+      "eval_generator_rougeL": 99.3289,
+      "eval_loss": 0.006375482305884361,
+      "eval_runtime": 611.1452,
+      "eval_samples_per_second": 5.586,
+      "eval_selector_accuracy": 95.3855,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 314
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.172842003562496e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fkl/checkpoint-314/training_args.bin b/qwen3/fkl/checkpoint-314/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..62f02c6f015c2ecc24752edd527d8a455bc9b089
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca0c534c8077d1140a5d674eead223cc4f38c64dc44b4464a6525db759412db
+size 7185
diff --git a/qwen3/fkl/checkpoint-314/zero_to_fp32.py b/qwen3/fkl/checkpoint-314/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fkl/checkpoint-314/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fkl/checkpoint-628/README.md b/qwen3/fkl/checkpoint-628/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-628/adapter_config.json b/qwen3/fkl/checkpoint-628/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ba50532007224c0205252929f5d7a29f112a16
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-628/adapter_model.safetensors b/qwen3/fkl/checkpoint-628/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..abec01b4d074878b6d9f7b032e5f5ade246b5297
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a49aa9ddc52ccd0cd9cf44b537b64b74243885c2308dde4fd1763f21ff9667d4
+size 40422560
diff --git a/qwen3/fkl/checkpoint-628/chat_template.jinja b/qwen3/fkl/checkpoint-628/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9059b72fc1adcd9222c1747aac4a43e38139a505
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db4554651f094e2892996fe2b4e577f35f38c8981a3a7f5ad4ab8f2128487790
+size 121135557
diff --git a/qwen3/fkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c3d6de5137ea3e77a16a91dbb7b2a5b80c3387d
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:680a9b9048d8471e47bd83e5d6e83a8326d6b5fe1cac43a434330c8d96ecb3f2
+size 121135749
diff --git a/qwen3/fkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/fkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..283adf1220c72a7b6fd7e9fba75ac1b1d7e3e15c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47d07926f822a457c3c66dd0f55bceb2c10161851b869954eab60447e94fe166
+size 351841765
diff --git a/qwen3/fkl/checkpoint-628/latest b/qwen3/fkl/checkpoint-628/latest
new file mode 100644
index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/latest
@@ -0,0 +1 @@
+global_step628
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-628/rng_state_0.pth b/qwen3/fkl/checkpoint-628/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a9b6118325bc42016f805af6ae0a1a8fcc6d2729
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:637fd8bd44c554f4717ddb9a69c2dcc1c02fb04b2dc311286326f9972b366059
+size 14917
diff --git a/qwen3/fkl/checkpoint-628/rng_state_1.pth b/qwen3/fkl/checkpoint-628/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..01c58e4f931176688a688181d695325dc58ad937
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:572d359a061f3dd0c5ca449ac8418b015557a177fbadf0209187848ff593b7bc
+size 14917
diff --git a/qwen3/fkl/checkpoint-628/scheduler.pt b/qwen3/fkl/checkpoint-628/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a
+size 1465
diff --git a/qwen3/fkl/checkpoint-628/tokenizer.json b/qwen3/fkl/checkpoint-628/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fkl/checkpoint-628/tokenizer_config.json b/qwen3/fkl/checkpoint-628/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fkl/checkpoint-628/trainer_state.json b/qwen3/fkl/checkpoint-628/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f3edfd3c537bf7e2b3bfcad0164c80ec9ded6a6
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/trainer_state.json
@@ -0,0 +1,281 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 628,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.7414703369140625,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.5651369094848633,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 0.7481251358985901,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.10692100524902344,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 0.8983101844787598,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.0871108591556549,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 0.3072285056114197,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.03171964287757874,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.5620023608207703,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.038917267322540285,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 0.34996840357780457,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.02393315136432648,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.285879522562027,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.0288031667470932,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 0.28871023654937744,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.020403537154197692,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 0.3916277289390564,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.017323681712150575,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.15098990499973297,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.022147342562675476,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.4012651741504669,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01552567332983017,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 0.48113393783569336,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.012960800528526306,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.6242109537124634,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.012793044745922088,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.29736602306365967,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.014466887712478638,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.320481538772583,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013739512860774994,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 91.764,
+      "eval_generator_rouge1": 99.6596,
+      "eval_generator_rouge2": 99.4475,
+      "eval_generator_rougeL": 99.3289,
+      "eval_loss": 0.006375482305884361,
+      "eval_runtime": 611.1452,
+      "eval_samples_per_second": 5.586,
+      "eval_selector_accuracy": 95.3855,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 1.464452862739563,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014014726877212525,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.08158843219280243,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.006687680631875992,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.12018018215894699,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.0065675824880599976,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.20543402433395386,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.004217476397752762,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.15771833062171936,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.010484220832586289,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.1818164736032486,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.009176531434059143,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.043983329087495804,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.004671639204025269,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.2847776710987091,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.003687678650021553,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.4654720425605774,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0022115066647529604,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.057350289076566696,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.006766787171363831,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.038868289440870285,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.007653500884771347,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.05846746638417244,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.00647004097700119,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.16134238243103027,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.0035313751548528673,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 2.057159423828125,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.008149225264787674,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 1.8178443908691406,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.00631907656788826,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.023604823276400566,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.003845648840069771,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 93.6916,
+      "eval_generator_rouge1": 99.7589,
+      "eval_generator_rouge2": 99.5845,
+      "eval_generator_rougeL": 99.5171,
+      "eval_loss": 0.004298548214137554,
+      "eval_runtime": 611.3037,
+      "eval_samples_per_second": 5.585,
+      "eval_selector_accuracy": 97.5467,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 628
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.346351770520781e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fkl/checkpoint-628/training_args.bin b/qwen3/fkl/checkpoint-628/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..62f02c6f015c2ecc24752edd527d8a455bc9b089
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca0c534c8077d1140a5d674eead223cc4f38c64dc44b4464a6525db759412db
+size 7185
diff --git a/qwen3/fkl/checkpoint-628/zero_to_fp32.py b/qwen3/fkl/checkpoint-628/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fkl/checkpoint-628/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fkl/checkpoint-942/README.md b/qwen3/fkl/checkpoint-942/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-942/adapter_config.json b/qwen3/fkl/checkpoint-942/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ba50532007224c0205252929f5d7a29f112a16
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-942/adapter_model.safetensors b/qwen3/fkl/checkpoint-942/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1811e1d5e90895c2dcf2ba5e2cd9b4c8e627c96b
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a3da92fea1f5924789c9315725bff2f8bbb4d755d7c3b5f80c496ee4b78d312
+size 40422560
diff --git a/qwen3/fkl/checkpoint-942/chat_template.jinja b/qwen3/fkl/checkpoint-942/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fd666b17d77f58b7bbcf03104d1866974aaeaab
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff997376c5128629b0fe66acb7067e77b0c0aa94d826a26113d0420c7278308e
+size 121135557
diff --git a/qwen3/fkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f09c55e47a2f204e049e11763cfa2cd536191da2
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:847fec49772c7f4a6a35d753f96e624d760d678e3e99aed3d7b410d086239596
+size 121135749
diff --git a/qwen3/fkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/fkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..301d99c5b7972175983bf4e1a404fa247b2e3fe6
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9416cc5c46c0b597d5ade9fc5176ed00f42e50eca9bd673f375baa197b616826
+size 351841765
diff --git a/qwen3/fkl/checkpoint-942/latest b/qwen3/fkl/checkpoint-942/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/latest
@@ -0,0 +1 @@
+global_step942
\ No newline at end of file
diff --git a/qwen3/fkl/checkpoint-942/rng_state_0.pth b/qwen3/fkl/checkpoint-942/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a973c2f491a1b041b3414bf278809a4063f61436
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0eb4343151d5c8060d318f215330972d0ebf23bd556ea7262eb9282f29fcf94
+size 14917
diff --git a/qwen3/fkl/checkpoint-942/rng_state_1.pth b/qwen3/fkl/checkpoint-942/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..44f5f6499eb6afb30ad2c8c655ffd61b9ac550c3
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1320de7e87c559c0ee99ef62711869ea643205b7e99ddc9dd4041c18f388f889
+size 14917
diff --git a/qwen3/fkl/checkpoint-942/scheduler.pt b/qwen3/fkl/checkpoint-942/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3
+size 1465
diff --git a/qwen3/fkl/checkpoint-942/tokenizer.json b/qwen3/fkl/checkpoint-942/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fkl/checkpoint-942/tokenizer_config.json b/qwen3/fkl/checkpoint-942/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fkl/checkpoint-942/trainer_state.json b/qwen3/fkl/checkpoint-942/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..7f250870d8270612c91d1b164298d7f5eed7013a
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/trainer_state.json
@@ -0,0 +1,408 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 942,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.7414703369140625,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.5651369094848633,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 0.7481251358985901,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.10692100524902344,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 0.8983101844787598,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.0871108591556549,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 0.3072285056114197,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.03171964287757874,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.5620023608207703,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.038917267322540285,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 0.34996840357780457,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.02393315136432648,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.285879522562027,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.0288031667470932,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 0.28871023654937744,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.020403537154197692,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 0.3916277289390564,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.017323681712150575,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.15098990499973297,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.022147342562675476,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.4012651741504669,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01552567332983017,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 0.48113393783569336,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.012960800528526306,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.6242109537124634,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.012793044745922088,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.29736602306365967,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.014466887712478638,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.320481538772583,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013739512860774994,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 91.764,
+      "eval_generator_rouge1": 99.6596,
+      "eval_generator_rouge2": 99.4475,
+      "eval_generator_rougeL": 99.3289,
+      "eval_loss": 0.006375482305884361,
+      "eval_runtime": 611.1452,
+      "eval_samples_per_second": 5.586,
+      "eval_selector_accuracy": 95.3855,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 1.464452862739563,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014014726877212525,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.08158843219280243,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.006687680631875992,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.12018018215894699,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.0065675824880599976,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.20543402433395386,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.004217476397752762,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.15771833062171936,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.010484220832586289,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.1818164736032486,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.009176531434059143,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.043983329087495804,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.004671639204025269,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.2847776710987091,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.003687678650021553,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.4654720425605774,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0022115066647529604,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.057350289076566696,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.006766787171363831,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.038868289440870285,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.007653500884771347,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.05846746638417244,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.00647004097700119,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.16134238243103027,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.0035313751548528673,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 2.057159423828125,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.008149225264787674,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 1.8178443908691406,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.00631907656788826,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.023604823276400566,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.003845648840069771,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 93.6916,
+      "eval_generator_rouge1": 99.7589,
+      "eval_generator_rouge2": 99.5845,
+      "eval_generator_rougeL": 99.5171,
+      "eval_loss": 0.004298548214137554,
+      "eval_runtime": 611.3037,
+      "eval_samples_per_second": 5.585,
+      "eval_selector_accuracy": 97.5467,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 628
+    },
+    {
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.07722756266593933,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.0037871412932872772,
+      "step": 640
+    },
+    {
+      "epoch": 2.102114080574392,
+      "grad_norm": 0.4329485297203064,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.006162230670452118,
+      "step": 660
+    },
+    {
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.15261922776699066,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.0018202636390924453,
+      "step": 680
+    },
+    {
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.031139470636844635,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.0023104183375835417,
+      "step": 700
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.018401559442281723,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.0016305617988109588,
+      "step": 720
+    },
+    {
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.03898131102323532,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.0027825018391013146,
+      "step": 740
+    },
+    {
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.008162672631442547,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.0016868755221366883,
+      "step": 760
+    },
+    {
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.05951504036784172,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0010992977768182755,
+      "step": 780
+    },
+    {
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.08524229377508163,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.0025967489928007126,
+      "step": 800
+    },
+    {
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.039385274052619934,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.0020601760596036913,
+      "step": 820
+    },
+    {
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.14205299317836761,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.002689024433493614,
+      "step": 840
+    },
+    {
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.07378510385751724,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.0022501811385154725,
+      "step": 860
+    },
+    {
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.16569221019744873,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.0014809093438088894,
+      "step": 880
+    },
+    {
+      "epoch": 2.8679696848823295,
+      "grad_norm": 0.12665939331054688,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.0022083733230829237,
+      "step": 900
+    },
+    {
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.005265355110168457,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.002069174498319626,
+      "step": 920
+    },
+    {
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.10703321546316147,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.0012829974293708801,
+      "step": 940
+    },
+    {
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.5023,
+      "eval_generator_rouge1": 99.9079,
+      "eval_generator_rouge2": 99.8141,
+      "eval_generator_rougeL": 99.6685,
+      "eval_loss": 0.00343098072335124,
+      "eval_runtime": 620.0123,
+      "eval_samples_per_second": 5.506,
+      "eval_selector_accuracy": 98.8318,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.173,
+      "step": 942
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.524157743353037e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fkl/checkpoint-942/training_args.bin b/qwen3/fkl/checkpoint-942/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..62f02c6f015c2ecc24752edd527d8a455bc9b089
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca0c534c8077d1140a5d674eead223cc4f38c64dc44b4464a6525db759412db
+size 7185
diff --git a/qwen3/fkl/checkpoint-942/zero_to_fp32.py b/qwen3/fkl/checkpoint-942/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/fkl/checkpoint-942/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/fkl/eval_results.json b/qwen3/fkl/eval_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..b962a5af1bf22d230191a287bab4d1b8c413d50e
--- /dev/null
+++ b/qwen3/fkl/eval_results.json
@@ -0,0 +1,14 @@
+{
+    "epoch": 5.0,
+    "eval_generator_count": 1712.0,
+    "eval_generator_exact_match": 96.9042,
+    "eval_generator_rouge1": 99.9327,
+    "eval_generator_rouge2": 99.876,
+    "eval_generator_rougeL": 99.7659,
+    "eval_loss": 0.0034750187769532204,
+    "eval_runtime": 599.1469,
+    "eval_samples_per_second": 5.698,
+    "eval_selector_accuracy": 98.6565,
+    "eval_selector_count": 1712.0,
+    "eval_steps_per_second": 0.179
+}
\ No newline at end of file
diff --git a/qwen3/fkl/tokenizer.json b/qwen3/fkl/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/fkl/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/fkl/tokenizer_config.json b/qwen3/fkl/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/fkl/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/fkl/train_results.json b/qwen3/fkl/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..f85b251ba4c38a8f14032b195eb5b2d93bb0d6c7
--- /dev/null
+++ b/qwen3/fkl/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 5.0,
+    "total_flos": 5.871444840310374e+16,
+    "train_loss": 0.015215813707275564,
+    "train_runtime": 8273.9277,
+    "train_samples_per_second": 6.059,
+    "train_steps_per_second": 0.19
+}
\ No newline at end of file
diff --git a/qwen3/fkl/trainer_state.json b/qwen3/fkl/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9244db01d7f9391d4e3ac3baa25c423e395db2d2
--- /dev/null
+++ b/qwen3/fkl/trainer_state.json
@@ -0,0 +1,664 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 1570,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06382130035899482,
+      "grad_norm": 2.7414703369140625,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.5651369094848633,
+      "step": 20
+    },
+    {
+      "epoch": 0.12764260071798963,
+      "grad_norm": 0.7481251358985901,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.10692100524902344,
+      "step": 40
+    },
+    {
+      "epoch": 0.19146390107698444,
+      "grad_norm": 0.8983101844787598,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.0871108591556549,
+      "step": 60
+    },
+    {
+      "epoch": 0.25528520143597927,
+      "grad_norm": 0.3072285056114197,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.03171964287757874,
+      "step": 80
+    },
+    {
+      "epoch": 0.3191065017949741,
+      "grad_norm": 0.5620023608207703,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.038917267322540285,
+      "step": 100
+    },
+    {
+      "epoch": 0.3829278021539689,
+      "grad_norm": 0.34996840357780457,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.02393315136432648,
+      "step": 120
+    },
+    {
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.285879522562027,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.0288031667470932,
+      "step": 140
+    },
+    {
+      "epoch": 0.5105704028719585,
+      "grad_norm": 0.28871023654937744,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.020403537154197692,
+      "step": 160
+    },
+    {
+      "epoch": 0.5743917032309533,
+      "grad_norm": 0.3916277289390564,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.017323681712150575,
+      "step": 180
+    },
+    {
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.15098990499973297,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.022147342562675476,
+      "step": 200
+    },
+    {
+      "epoch": 0.702034303948943,
+      "grad_norm": 0.4012651741504669,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01552567332983017,
+      "step": 220
+    },
+    {
+      "epoch": 0.7658556043079378,
+      "grad_norm": 0.48113393783569336,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.012960800528526306,
+      "step": 240
+    },
+    {
+      "epoch": 0.8296769046669326,
+      "grad_norm": 0.6242109537124634,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.012793044745922088,
+      "step": 260
+    },
+    {
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.29736602306365967,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.014466887712478638,
+      "step": 280
+    },
+    {
+      "epoch": 0.9573195053849222,
+      "grad_norm": 0.320481538772583,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013739512860774994,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 91.764,
+      "eval_generator_rouge1": 99.6596,
+      "eval_generator_rouge2": 99.4475,
+      "eval_generator_rougeL": 99.3289,
+      "eval_loss": 0.006375482305884361,
+      "eval_runtime": 611.1452,
+      "eval_samples_per_second": 5.586,
+      "eval_selector_accuracy": 95.3855,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 314
+    },
+    {
+      "epoch": 1.0191463901076985,
+      "grad_norm": 1.464452862739563,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014014726877212525,
+      "step": 320
+    },
+    {
+      "epoch": 1.0829676904666932,
+      "grad_norm": 0.08158843219280243,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.006687680631875992,
+      "step": 340
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.12018018215894699,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.0065675824880599976,
+      "step": 360
+    },
+    {
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.20543402433395386,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.004217476397752762,
+      "step": 380
+    },
+    {
+      "epoch": 1.2744315915436777,
+      "grad_norm": 0.15771833062171936,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.010484220832586289,
+      "step": 400
+    },
+    {
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.1818164736032486,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.009176531434059143,
+      "step": 420
+    },
+    {
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.043983329087495804,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.004671639204025269,
+      "step": 440
+    },
+    {
+      "epoch": 1.4658954926206622,
+      "grad_norm": 0.2847776710987091,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.003687678650021553,
+      "step": 460
+    },
+    {
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.4654720425605774,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0022115066647529604,
+      "step": 480
+    },
+    {
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.057350289076566696,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.006766787171363831,
+      "step": 500
+    },
+    {
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.038868289440870285,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.007653500884771347,
+      "step": 520
+    },
+    {
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.05846746638417244,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.00647004097700119,
+      "step": 540
+    },
+    {
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.16134238243103027,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.0035313751548528673,
+      "step": 560
+    },
+    {
+      "epoch": 1.848823294774631,
+      "grad_norm": 2.057159423828125,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.008149225264787674,
+      "step": 580
+    },
+    {
+      "epoch": 1.9126445951336257,
+      "grad_norm": 1.8178443908691406,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.00631907656788826,
+      "step": 600
+    },
+    {
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.023604823276400566,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.003845648840069771,
+      "step": 620
+    },
+    {
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 93.6916,
+      "eval_generator_rouge1": 99.7589,
+      "eval_generator_rouge2": 99.5845,
+      "eval_generator_rougeL": 99.5171,
+      "eval_loss": 0.004298548214137554,
+      "eval_runtime": 611.3037,
+      "eval_samples_per_second": 5.585,
+      "eval_selector_accuracy": 97.5467,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.175,
+      "step": 628
+    },
+    {
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.07722756266593933,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.0037871412932872772,
+      "step": 640
+    },
+    {
+      "epoch": 2.102114080574392,
+      "grad_norm": 0.4329485297203064,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.006162230670452118,
+      "step": 660
+    },
+    {
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.15261922776699066,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.0018202636390924453,
+      "step": 680
+    },
+    {
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.031139470636844635,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.0023104183375835417,
+      "step": 700
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.018401559442281723,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.0016305617988109588,
+      "step": 720
+    },
+    {
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.03898131102323532,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.0027825018391013146,
+      "step": 740
+    },
+    {
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.008162672631442547,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.0016868755221366883,
+      "step": 760
+    },
+    {
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.05951504036784172,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0010992977768182755,
+      "step": 780
+    },
+    {
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.08524229377508163,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.0025967489928007126,
+      "step": 800
+    },
+    {
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.039385274052619934,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.0020601760596036913,
+      "step": 820
+    },
+    {
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.14205299317836761,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.002689024433493614,
+      "step": 840
+    },
+    {
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.07378510385751724,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.0022501811385154725,
+      "step": 860
+    },
+    {
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.16569221019744873,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.0014809093438088894,
+      "step": 880
+    },
+    {
+      "epoch": 2.8679696848823295,
+      "grad_norm": 0.12665939331054688,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.0022083733230829237,
+      "step": 900
+    },
+    {
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.005265355110168457,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.002069174498319626,
+      "step": 920
+    },
+    {
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.10703321546316147,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.0012829974293708801,
+      "step": 940
+    },
+    {
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.5023,
+      "eval_generator_rouge1": 99.9079,
+      "eval_generator_rouge2": 99.8141,
+      "eval_generator_rougeL": 99.6685,
+      "eval_loss": 0.00343098072335124,
+      "eval_runtime": 620.0123,
+      "eval_samples_per_second": 5.506,
+      "eval_selector_accuracy": 98.8318,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.173,
+      "step": 942
+    },
+    {
+      "epoch": 3.057439170323095,
+      "grad_norm": 0.023057373240590096,
+      "learning_rate": 3.3007844105971336e-05,
+      "loss": 0.0013455232605338097,
+      "step": 960
+    },
+    {
+      "epoch": 3.12126047068209,
+      "grad_norm": 0.006213244516402483,
+      "learning_rate": 3.114292593987149e-05,
+      "loss": 0.001208091713488102,
+      "step": 980
+    },
+    {
+      "epoch": 3.185081771041085,
+      "grad_norm": 0.0030793482437729836,
+      "learning_rate": 2.9308285746383408e-05,
+      "loss": 0.0011879941448569298,
+      "step": 1000
+    },
+    {
+      "epoch": 3.24890307140008,
+      "grad_norm": 0.025737427175045013,
+      "learning_rate": 2.7506861536187867e-05,
+      "loss": 0.0008374839089810848,
+      "step": 1020
+    },
+    {
+      "epoch": 3.3127243717590744,
+      "grad_norm": 0.03453245759010315,
+      "learning_rate": 2.5741538127566396e-05,
+      "loss": 0.0008559312671422958,
+      "step": 1040
+    },
+    {
+      "epoch": 3.3765456721180693,
+      "grad_norm": 0.001912926440127194,
+      "learning_rate": 2.4015142526625115e-05,
+      "loss": 0.0016484295949339867,
+      "step": 1060
+    },
+    {
+      "epoch": 3.4403669724770642,
+      "grad_norm": 0.07635863870382309,
+      "learning_rate": 2.233043940009967e-05,
+      "loss": 0.0025718800723552706,
+      "step": 1080
+    },
+    {
+      "epoch": 3.504188272836059,
+      "grad_norm": 0.015918560326099396,
+      "learning_rate": 2.0690126647990974e-05,
+      "loss": 0.0009869973175227642,
+      "step": 1100
+    },
+    {
+      "epoch": 3.568009573195054,
+      "grad_norm": 0.014826856553554535,
+      "learning_rate": 1.9096831083121828e-05,
+      "loss": 0.0014582538977265358,
+      "step": 1120
+    },
+    {
+      "epoch": 3.6318308735540485,
+      "grad_norm": 0.020788000896573067,
+      "learning_rate": 1.7553104224533324e-05,
+      "loss": 0.000823317002505064,
+      "step": 1140
+    },
+    {
+      "epoch": 3.6956521739130435,
+      "grad_norm": 0.005068339407444,
+      "learning_rate": 1.606141821145735e-05,
+      "loss": 0.001963364891707897,
+      "step": 1160
+    },
+    {
+      "epoch": 3.7594734742720384,
+      "grad_norm": 0.012145614251494408,
+      "learning_rate": 1.4624161844408763e-05,
+      "loss": 0.0021551501005887983,
+      "step": 1180
+    },
+    {
+      "epoch": 3.823294774631033,
+      "grad_norm": 0.011486915871500969,
+      "learning_rate": 1.324363675973698e-05,
+      "loss": 0.0011032622307538987,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8871160749900278,
+      "grad_norm": 0.06560127437114716,
+      "learning_rate": 1.1922053743763052e-05,
+      "loss": 0.0013195869512856006,
+      "step": 1220
+    },
+    {
+      "epoch": 3.9509373753490227,
+      "grad_norm": 0.014271366409957409,
+      "learning_rate": 1.0661529192404973e-05,
+      "loss": 0.0009616646915674209,
+      "step": 1240
+    },
+    {
+      "epoch": 4.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.3785,
+      "eval_generator_rouge1": 99.9022,
+      "eval_generator_rouge2": 99.8287,
+      "eval_generator_rougeL": 99.7265,
+      "eval_loss": 0.0034609222784638405,
+      "eval_runtime": 688.7408,
+      "eval_samples_per_second": 4.957,
+      "eval_selector_accuracy": 98.5397,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.155,
+      "step": 1256
+    },
+    {
+      "epoch": 4.0127642600717985,
+      "grad_norm": 0.026660269126296043,
+      "learning_rate": 9.464081721960592e-06,
+      "loss": 0.0014410967007279396,
+      "step": 1260
+    },
+    {
+      "epoch": 4.076585560430794,
+      "grad_norm": 0.055254511535167694,
+      "learning_rate": 8.331628936475818e-06,
+      "loss": 0.0007118118926882743,
+      "step": 1280
+    },
+    {
+      "epoch": 4.140406860789788,
+      "grad_norm": 0.0038272945675998926,
+      "learning_rate": 7.2659843568748385e-06,
+      "loss": 0.0010564759373664855,
+      "step": 1300
+    },
+    {
+      "epoch": 4.204228161148784,
+      "grad_norm": 0.009401796385645866,
+      "learning_rate": 6.268854516770008e-06,
+      "loss": 0.0007780813612043858,
+      "step": 1320
+    },
+    {
+      "epoch": 4.268049461507778,
+      "grad_norm": 0.03749614953994751,
+      "learning_rate": 5.341836229602331e-06,
+      "loss": 0.0010510680265724659,
+      "step": 1340
+    },
+    {
+      "epoch": 4.331870761866773,
+      "grad_norm": 0.008090543560683727,
+      "learning_rate": 4.4864140314888216e-06,
+      "loss": 0.0007310851942747831,
+      "step": 1360
+    },
+    {
+      "epoch": 4.395692062225768,
+      "grad_norm": 0.003737018443644047,
+      "learning_rate": 3.703957803871932e-06,
+      "loss": 0.000812545046210289,
+      "step": 1380
+    },
+    {
+      "epoch": 4.4595133625847625,
+      "grad_norm": 0.03762929514050484,
+      "learning_rate": 2.9957205797780495e-06,
+      "loss": 0.0015857372432947158,
+      "step": 1400
+    },
+    {
+      "epoch": 4.523334662943758,
+      "grad_norm": 0.04096720740199089,
+      "learning_rate": 2.362836537198199e-06,
+      "loss": 0.001064287591725588,
+      "step": 1420
+    },
+    {
+      "epoch": 4.587155963302752,
+      "grad_norm": 0.015219061635434628,
+      "learning_rate": 1.8063191828043703e-06,
+      "loss": 0.0007133203092962503,
+      "step": 1440
+    },
+    {
+      "epoch": 4.650977263661747,
+      "grad_norm": 0.019106416031718254,
+      "learning_rate": 1.3270597289100437e-06,
+      "loss": 0.0007402435876429081,
+      "step": 1460
+    },
+    {
+      "epoch": 4.714798564020742,
+      "grad_norm": 0.03157809376716614,
+      "learning_rate": 9.258256662741205e-07,
+      "loss": 0.0008887875825166702,
+      "step": 1480
+    },
+    {
+      "epoch": 4.778619864379737,
+      "grad_norm": 0.008441638201475143,
+      "learning_rate": 6.032595350337341e-07,
+      "loss": 0.0012488246895372868,
+      "step": 1500
+    },
+    {
+      "epoch": 4.842441164738732,
+      "grad_norm": 0.027420049533247948,
+      "learning_rate": 3.5987789573421356e-07,
+      "loss": 0.004218760132789612,
+      "step": 1520
+    },
+    {
+      "epoch": 4.9062624650977265,
+      "grad_norm": 1.1041139364242554,
+      "learning_rate": 1.9607050210401192e-07,
+      "loss": 0.001393320318311453,
+      "step": 1540
+    },
+    {
+      "epoch": 4.970083765456721,
+      "grad_norm": 0.025242790579795837,
+      "learning_rate": 1.1209967689925832e-07,
+      "loss": 0.0008278143592178822,
+      "step": 1560
+    },
+    {
+      "epoch": 5.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.9696,
+      "eval_generator_rouge1": 99.9236,
+      "eval_generator_rouge2": 99.8518,
+      "eval_generator_rougeL": 99.7241,
+      "eval_loss": 0.0034750187769532204,
+      "eval_runtime": 712.376,
+      "eval_samples_per_second": 4.792,
+      "eval_selector_accuracy": 98.6565,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.15,
+      "step": 1570
+    },
+    {
+      "epoch": 5.0,
+      "step": 1570,
+      "total_flos": 5.871444840310374e+16,
+      "train_loss": 0.015215813707275564,
+      "train_runtime": 8273.9277,
+      "train_samples_per_second": 6.059,
+      "train_steps_per_second": 0.19
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.871444840310374e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/fkl/training_args.bin b/qwen3/fkl/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..62f02c6f015c2ecc24752edd527d8a455bc9b089
--- /dev/null
+++ b/qwen3/fkl/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca0c534c8077d1140a5d674eead223cc4f38c64dc44b4464a6525db759412db
+size 7185
diff --git a/qwen3/hpd/README.md b/qwen3/hpd/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..3efd98fee279357b9e3db654d79afa8ea3474d27
--- /dev/null
+++ b/qwen3/hpd/README.md
@@ -0,0 +1,79 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen3-0.6B
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: hpd
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# hpd
+
+This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0033
+- Selector Count: 1712.0
+- Generator Count: 1712.0
+- Selector Accuracy: 98.7734
+- Generator Exact Match: 96.6121
+- Generator Rouge1: 99.8409
+- Generator Rouge2: 99.7633
+- Generator Rougel: 99.6798
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 2
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_min_lr
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel | Student Nll Loss | Loss   | Diff Pct | Neg Pct | Active Pct | Pct    | Gt Raw Mean | Gt Raw Min | Gt Raw Max | Gt Raw Neg Pct | Sample Mean | Sample Min | Sample Max | Sample Neg Pct |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:|:----------------:|:------:|:--------:|:-------:|:----------:|:------:|:-----------:|:----------:|:----------:|:--------------:|:-----------:|:----------:|:----------:|:--------------:|
+| 0.0139        | 1.0   | 314  | 0.0101          | 1712.0         | 1712.0          | 93.9836           | 86.6822               | 99.4006          | 99.0209          | 98.951           | 0.0197           | 0.0227 | 0.8597   | 0.8509  | 0.8597     | 0.8597 | 0.0055      | -0.0054    | 0.1417     | 0.8509         | -1.6359     | -1.8393    | -1.4249    | 36.4486        |
+| 0.0021        | 2.0   | 628  | 0.0047          | 1712.0         | 1712.0          | 98.014            | 95.1519               | 99.7488          | 99.6229          | 99.5782          | 0.0014           | 0.0012 | 0.1079   | 3.3907  | 0.1079     | 0.1079 | 0.0007      | -0.0028    | 0.0358     | 3.3907         | -0.0696     | -0.0701    | -0.0692    | 10.1695        |
+| 0.0011        | 3.0   | 942  | 0.0040          | 1712.0         | 1712.0          | 97.7804           | 95.6776               | 99.8328          | 99.7335          | 99.6469          | 0.0014           | 0.0012 | 0.1100   | 5.0010  | 0.1100     | 0.1100 | 0.0007      | -0.0005    | 0.0329     | 5.0010         | -0.4341     | -0.4341    | -0.4341    | 9.0909         |
+| 0.0007        | 4.0   | 1256 | 0.0034          | 1712.0         | 1712.0          | 98.5981           | 96.4953               | 99.8312          | 99.7559          | 99.645           | 0.0015           | 0.0013 | 0.0877   | 7.9339  | 0.0740     | 0.0740 | 0.0003      | -0.0034    | 0.0194     | 7.9339         | -0.0697     | -0.0697    | -0.0697    | 6.9106         |
+| 0.0013        | 5.0   | 1570 | 0.0033          | 1712.0         | 1712.0          | 98.5981           | 96.4953               | 99.897           | 99.8245          | 99.7185          | 0.0007           | 0.0004 | 0.0610   | 8.7527  | 0.0506     | 0.0506 | 0.0002      | -0.0047    | 0.0160     | 8.7527         | -0.0083     | -0.0083    | -0.0083    | 4.6667         |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.8.0
+- Pytorch 2.9.1+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2
\ No newline at end of file
diff --git a/qwen3/hpd/adapter_config.json b/qwen3/hpd/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0f6594b49d741b9fe4eb4a1a4df0b797e15bcd
--- /dev/null
+++ b/qwen3/hpd/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/hpd/adapter_model.safetensors b/qwen3/hpd/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3a366629662cf5ee4469936aa378a672e2f2651e
--- /dev/null
+++ b/qwen3/hpd/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aaf8b6a7a4f0e6d4b7fbe24b411c1a83ffd5bd908302ea9c83a79f8b006567e
+size 40422560
diff --git a/qwen3/hpd/all_results.json b/qwen3/hpd/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..45df431fc9910b30fb1277c2a3814e3c23a6b6ec
--- /dev/null
+++ b/qwen3/hpd/all_results.json
@@ -0,0 +1,19 @@
+{
+    "epoch": 5.0,
+    "eval_generator_count": 1712.0,
+    "eval_generator_exact_match": 96.6121,
+    "eval_generator_rouge1": 99.8409,
+    "eval_generator_rouge2": 99.7633,
+    "eval_generator_rougeL": 99.6798,
+    "eval_loss": 0.0032775094732642174,
+    "eval_runtime": 579.5375,
+    "eval_samples_per_second": 5.891,
+    "eval_selector_accuracy": 98.7734,
+    "eval_selector_count": 1712.0,
+    "eval_steps_per_second": 0.185,
+    "total_flos": 5.871444840310374e+16,
+    "train_loss": 0.018279938655608827,
+    "train_runtime": 8124.6936,
+    "train_samples_per_second": 6.171,
+    "train_steps_per_second": 0.193
+}
\ No newline at end of file
diff --git a/qwen3/hpd/chat_template.jinja b/qwen3/hpd/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/hpd/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1256/README.md b/qwen3/hpd/checkpoint-1256/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1256/adapter_config.json b/qwen3/hpd/checkpoint-1256/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0f6594b49d741b9fe4eb4a1a4df0b797e15bcd
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1256/adapter_model.safetensors b/qwen3/hpd/checkpoint-1256/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dbbb1d7cefa64e905c75da863a3e3178ba476b12
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:466d97f87e7664a10c3039c5d8aa72fe9857a288f88e4953def0101807e84beb
+size 40422560
diff --git a/qwen3/hpd/checkpoint-1256/chat_template.jinja b/qwen3/hpd/checkpoint-1256/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..794669c60c90eeb842ff36f746fe6d76253ba126
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7d1b8d95a552fdf75f65923dfcab26d78b8c4d96873d4ba523dae04954d5336
+size 121135557
diff --git a/qwen3/hpd/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd3693ea14051c227e8e60783d2d1e8b274fdf55
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37a67724a8a68e871231f15b6d8c159a2b95e77e3fed9f8a5abaf965a6e9052c
+size 121135749
diff --git a/qwen3/hpd/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/hpd/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7adce16820decadf6ca8787c91b9a0bab838904
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01529d4281f8b7ecbab9875b7121c3ff14076a0177d1f0aa4a8b5743f8bd8b8d
+size 351841765
diff --git a/qwen3/hpd/checkpoint-1256/latest b/qwen3/hpd/checkpoint-1256/latest
new file mode 100644
index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/latest
@@ -0,0 +1 @@
+global_step1256
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1256/rng_state_0.pth b/qwen3/hpd/checkpoint-1256/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..0e5d13edcf2d7af9dccd48b5246553bd5231ea1b
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34bf9df54f681f008a2a7f065e276d5fd9374d468c2da02fcd34a3fe48d0ab03
+size 14917
diff --git a/qwen3/hpd/checkpoint-1256/rng_state_1.pth b/qwen3/hpd/checkpoint-1256/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..cf9f19a581f3875b6f082d8a8716578345db344b
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90b89e7477fc528926c21e7fcb6f80f900260a2286d5430bdc62587d587faad0
+size 14917
diff --git a/qwen3/hpd/checkpoint-1256/scheduler.pt b/qwen3/hpd/checkpoint-1256/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23
+size 1465
diff --git a/qwen3/hpd/checkpoint-1256/tokenizer.json b/qwen3/hpd/checkpoint-1256/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/hpd/checkpoint-1256/tokenizer_config.json b/qwen3/hpd/checkpoint-1256/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/hpd/checkpoint-1256/trainer_state.json b/qwen3/hpd/checkpoint-1256/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..32e9de9b25047a576da77e138e7603604c5a0126
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/trainer_state.json
@@ -0,0 +1,1518 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 1256,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "adv1_neg_pct": 0.2358350306749344,
+      "adv2_active_pct": 17.716575622558594,
+      "ce_loss": 0.6496895551681519,
+      "epoch": 0.06382130035899482,
+      "grad_norm": 4.9955339431762695,
+      "hpd_loss": 0.7524586915969849,
+      "k1_gt_raw_max": 0.3395599126815796,
+      "k1_gt_raw_mean": 0.08363302797079086,
+      "k1_gt_raw_min": -0.006161125842481852,
+      "k1_gt_raw_neg_pct": 0.2358350306749344,
+      "k1_sample_max": -1.2287571430206299,
+      "k1_sample_mean": -5.443553924560547,
+      "k1_sample_min": -17.516345977783203,
+      "k1_sample_neg_pct": 97.79017639160156,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.7524587631225585,
+      "mask3_pct": 17.716575622558594,
+      "neg_student_nll_loss": 0.6496895551681519,
+      "sampled_diff_pct": 17.718591690063477,
+      "step": 20
+    },
+    {
+      "adv1_neg_pct": 0.24569129943847656,
+      "adv2_active_pct": 6.35067081451416,
+      "ce_loss": 0.12904959917068481,
+      "epoch": 0.12764260071798963,
+      "grad_norm": 3.5179519653320312,
+      "hpd_loss": 0.07163389772176743,
+      "k1_gt_raw_max": 0.3387124538421631,
+      "k1_gt_raw_mean": 0.03725019469857216,
+      "k1_gt_raw_min": -0.009099474176764488,
+      "k1_gt_raw_neg_pct": 0.24569129943847656,
+      "k1_sample_max": -1.1939605474472046,
+      "k1_sample_mean": -4.561788082122803,
+      "k1_sample_min": -10.153608322143555,
+      "k1_sample_neg_pct": 96.875,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.07163387537002563,
+      "mask3_pct": 6.35067081451416,
+      "neg_student_nll_loss": 0.12904959917068481,
+      "sampled_diff_pct": 6.35067081451416,
+      "step": 40
+    },
+    {
+      "adv1_neg_pct": 0.18458101153373718,
+      "adv2_active_pct": 5.008382320404053,
+      "ce_loss": 0.10459579527378082,
+      "epoch": 0.19146390107698444,
+      "grad_norm": 2.890872001647949,
+      "hpd_loss": 0.09348365664482117,
+      "k1_gt_raw_max": 0.3109760880470276,
+      "k1_gt_raw_mean": 0.026082728058099747,
+      "k1_gt_raw_min": -0.01705370843410492,
+      "k1_gt_raw_neg_pct": 0.18458101153373718,
+      "k1_sample_max": -2.0660922527313232,
+      "k1_sample_mean": -4.749420166015625,
+      "k1_sample_min": -8.562150955200195,
+      "k1_sample_neg_pct": 89.6875,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.09348366260528565,
+      "mask3_pct": 5.008382320404053,
+      "neg_student_nll_loss": 0.10459579527378082,
+      "sampled_diff_pct": 5.008382320404053,
+      "step": 60
+    },
+    {
+      "adv1_neg_pct": 0.3009452819824219,
+      "adv2_active_pct": 3.9314260482788086,
+      "ce_loss": 0.08063317835330963,
+      "epoch": 0.25528520143597927,
+      "grad_norm": 2.033250570297241,
+      "hpd_loss": 0.08107596635818481,
+      "k1_gt_raw_max": 0.2881737947463989,
+      "k1_gt_raw_mean": 0.019415946677327156,
+      "k1_gt_raw_min": -0.008919830434024334,
+      "k1_gt_raw_neg_pct": 0.3009452819824219,
+      "k1_sample_max": -1.6159262657165527,
+      "k1_sample_mean": -3.7506155967712402,
+      "k1_sample_min": -6.360409259796143,
+      "k1_sample_neg_pct": 86.39583587646484,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.0810759723186493,
+      "mask3_pct": 3.9314260482788086,
+      "neg_student_nll_loss": 0.08063317835330963,
+      "sampled_diff_pct": 3.9378738403320312,
+      "step": 80
+    },
+    {
+      "adv1_neg_pct": 0.22911369800567627,
+      "adv2_active_pct": 2.694115161895752,
+      "ce_loss": 0.052447859197854996,
+      "epoch": 0.3191065017949741,
+      "grad_norm": 1.751448392868042,
+      "hpd_loss": 0.027988094836473465,
+      "k1_gt_raw_max": 0.3031776547431946,
+      "k1_gt_raw_mean": 0.01864592172205448,
+      "k1_gt_raw_min": -0.007533406838774681,
+      "k1_gt_raw_neg_pct": 0.22911369800567627,
+      "k1_sample_max": -1.57374906539917,
+      "k1_sample_mean": -3.1450629234313965,
+      "k1_sample_min": -5.145340919494629,
+      "k1_sample_neg_pct": 84.70052337646484,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.027988094091415405,
+      "mask3_pct": 2.6920177936553955,
+      "neg_student_nll_loss": 0.052447859197854996,
+      "sampled_diff_pct": 2.7128100395202637,
+      "step": 100
+    },
+    {
+      "adv1_neg_pct": 0.1952832043170929,
+      "adv2_active_pct": 3.306915044784546,
+      "ce_loss": 0.04980571195483208,
+      "epoch": 0.3829278021539689,
+      "grad_norm": 3.930035352706909,
+      "hpd_loss": 0.015151199884712696,
+      "k1_gt_raw_max": 0.28564560413360596,
+      "k1_gt_raw_mean": 0.01884515956044197,
+      "k1_gt_raw_min": -0.001684204675257206,
+      "k1_gt_raw_neg_pct": 0.1952832043170929,
+      "k1_sample_max": -1.9181127548217773,
+      "k1_sample_mean": -3.2553763389587402,
+      "k1_sample_min": -4.8449602127075195,
+      "k1_sample_neg_pct": 75.20833587646484,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.015151196718215942,
+      "mask3_pct": 3.306915044784546,
+      "neg_student_nll_loss": 0.04980571195483208,
+      "sampled_diff_pct": 3.3092129230499268,
+      "step": 120
+    },
+    {
+      "adv1_neg_pct": 0.3264433741569519,
+      "adv2_active_pct": 2.25433087348938,
+      "ce_loss": 0.04018855467438698,
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.6639359593391418,
+      "hpd_loss": 0.02619136869907379,
+      "k1_gt_raw_max": 0.24626323580741882,
+      "k1_gt_raw_mean": 0.015010751783847809,
+      "k1_gt_raw_min": -0.004701526835560799,
+      "k1_gt_raw_neg_pct": 0.3264433741569519,
+      "k1_sample_max": -1.902454137802124,
+      "k1_sample_mean": -2.805896282196045,
+      "k1_sample_min": -3.829012393951416,
+      "k1_sample_neg_pct": 71.25,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.026191365718841553,
+      "mask3_pct": 2.25433087348938,
+      "neg_student_nll_loss": 0.04018855467438698,
+      "sampled_diff_pct": 2.2565953731536865,
+      "step": 140
+    },
+    {
+      "adv1_neg_pct": 0.3137112259864807,
+      "adv2_active_pct": 1.3726557493209839,
+      "ce_loss": 0.03148110955953598,
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.548140525817871,
+      "hpd_loss": 0.03039727918803692,
+      "k1_gt_raw_max": 0.20344290137290955,
+      "k1_gt_raw_mean": 0.009032711386680603,
+      "k1_gt_raw_min": -0.0009857998229563236,
+      "k1_gt_raw_neg_pct": 0.3137112259864807,
+      "k1_sample_max": -2.153831958770752,
+      "k1_sample_mean": -2.81235933303833,
+      "k1_sample_min": -3.547999858856201,
+      "k1_sample_neg_pct": 58.75,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.03039727509021759,
+      "mask3_pct": 1.3726557493209839,
+      "neg_student_nll_loss": 0.03148110955953598,
+      "sampled_diff_pct": 1.3726557493209839,
+      "step": 160
+    },
+    {
+      "adv1_neg_pct": 0.4895250201225281,
+      "adv2_active_pct": 1.1766502857208252,
+      "ce_loss": 0.02162190154194832,
+      "epoch": 0.5743917032309533,
+      "grad_norm": 2.528654098510742,
+      "hpd_loss": 0.011955421417951584,
+      "k1_gt_raw_max": 0.20187309384346008,
+      "k1_gt_raw_mean": 0.008551979437470436,
+      "k1_gt_raw_min": -0.0070910281501710415,
+      "k1_gt_raw_neg_pct": 0.4895250201225281,
+      "k1_sample_max": -1.979987621307373,
+      "k1_sample_mean": -2.5711493492126465,
+      "k1_sample_min": -3.2561821937561035,
+      "k1_sample_neg_pct": 56.09375,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.011955419182777404,
+      "mask3_pct": 1.1766502857208252,
+      "neg_student_nll_loss": 0.02162190154194832,
+      "sampled_diff_pct": 1.185581922531128,
+      "step": 180
+    },
+    {
+      "adv1_neg_pct": 0.5023188591003418,
+      "adv2_active_pct": 1.5687706470489502,
+      "ce_loss": 0.027880553156137466,
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.9650227427482605,
+      "hpd_loss": 0.010340285487473011,
+      "k1_gt_raw_max": 0.2007327377796173,
+      "k1_gt_raw_mean": 0.009667925536632538,
+      "k1_gt_raw_min": -0.01589183136820793,
+      "k1_gt_raw_neg_pct": 0.5023188591003418,
+      "k1_sample_max": -1.5354162454605103,
+      "k1_sample_mean": -2.1268630027770996,
+      "k1_sample_min": -2.771559238433838,
+      "k1_sample_neg_pct": 56.40625,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.010340283811092376,
+      "mask3_pct": 1.5687706470489502,
+      "neg_student_nll_loss": 0.027880553156137466,
+      "sampled_diff_pct": 1.5725816488265991,
+      "step": 200
+    },
+    {
+      "adv1_neg_pct": 0.48300591111183167,
+      "adv2_active_pct": 1.1448569297790527,
+      "ce_loss": 0.023295864462852478,
+      "epoch": 0.702034303948943,
+      "grad_norm": 2.59177303314209,
+      "hpd_loss": 0.019972147420048714,
+      "k1_gt_raw_max": 0.18090441823005676,
+      "k1_gt_raw_mean": 0.0075406162068247795,
+      "k1_gt_raw_min": -0.014693280681967735,
+      "k1_gt_raw_neg_pct": 0.48300591111183167,
+      "k1_sample_max": -1.6416490077972412,
+      "k1_sample_mean": -2.2296323776245117,
+      "k1_sample_min": -2.930915355682373,
+      "k1_sample_neg_pct": 48.515625,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01997213959693909,
+      "mask3_pct": 1.1448569297790527,
+      "neg_student_nll_loss": 0.023295864462852478,
+      "sampled_diff_pct": 1.154510736465454,
+      "step": 220
+    },
+    {
+      "adv1_neg_pct": 0.765082597732544,
+      "adv2_active_pct": 0.9244414567947388,
+      "ce_loss": 0.02368738129734993,
+      "epoch": 0.7658556043079378,
+      "grad_norm": 1.1686080694198608,
+      "hpd_loss": 0.02641882374882698,
+      "k1_gt_raw_max": 0.1654866635799408,
+      "k1_gt_raw_mean": 0.005874675698578358,
+      "k1_gt_raw_min": -0.009244760498404503,
+      "k1_gt_raw_neg_pct": 0.765082597732544,
+      "k1_sample_max": -1.3134702444076538,
+      "k1_sample_mean": -1.9339661598205566,
+      "k1_sample_min": -2.6181559562683105,
+      "k1_sample_neg_pct": 45.3125,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.026418817043304444,
+      "mask3_pct": 0.9244414567947388,
+      "neg_student_nll_loss": 0.02368738129734993,
+      "sampled_diff_pct": 0.9244414567947388,
+      "step": 240
+    },
+    {
+      "adv1_neg_pct": 0.8438892364501953,
+      "adv2_active_pct": 0.929736852645874,
+      "ce_loss": 0.020094729959964752,
+      "epoch": 0.8296769046669326,
+      "grad_norm": 1.2779905796051025,
+      "hpd_loss": 0.015618223696947098,
+      "k1_gt_raw_max": 0.16853322088718414,
+      "k1_gt_raw_mean": 0.006190822925418615,
+      "k1_gt_raw_min": -0.009831264615058899,
+      "k1_gt_raw_neg_pct": 0.8438892364501953,
+      "k1_sample_max": -1.3172353506088257,
+      "k1_sample_mean": -1.7796552181243896,
+      "k1_sample_min": -2.261643409729004,
+      "k1_sample_neg_pct": 44.95738983154297,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.015618222951889037,
+      "mask3_pct": 0.9239488840103149,
+      "neg_student_nll_loss": 0.020094729959964752,
+      "sampled_diff_pct": 0.9378187656402588,
+      "step": 260
+    },
+    {
+      "adv1_neg_pct": 0.9945487976074219,
+      "adv2_active_pct": 0.7392160892486572,
+      "ce_loss": 0.015887977555394173,
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.26237931847572327,
+      "hpd_loss": 0.015452454797923565,
+      "k1_gt_raw_max": 0.14525339007377625,
+      "k1_gt_raw_mean": 0.0046372562646865845,
+      "k1_gt_raw_min": -0.003686224576085806,
+      "k1_gt_raw_neg_pct": 0.9945487976074219,
+      "k1_sample_max": -1.206074595451355,
+      "k1_sample_mean": -1.4332499504089355,
+      "k1_sample_min": -1.671927571296692,
+      "k1_sample_neg_pct": 35.625,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.015452450513839722,
+      "mask3_pct": 0.7392160892486572,
+      "neg_student_nll_loss": 0.015887977555394173,
+      "sampled_diff_pct": 0.7392160892486572,
+      "step": 280
+    },
+    {
+      "adv1_neg_pct": 0.9700182676315308,
+      "adv2_active_pct": 0.6018879413604736,
+      "ce_loss": 0.014513559639453888,
+      "epoch": 0.9573195053849222,
+      "grad_norm": 2.0338079929351807,
+      "hpd_loss": 0.013913260772824287,
+      "k1_gt_raw_max": 0.13846856355667114,
+      "k1_gt_raw_mean": 0.004541991278529167,
+      "k1_gt_raw_min": -0.002533096820116043,
+      "k1_gt_raw_neg_pct": 0.9700182676315308,
+      "k1_sample_max": -1.2417628765106201,
+      "k1_sample_mean": -1.4318351745605469,
+      "k1_sample_min": -1.6163195371627808,
+      "k1_sample_neg_pct": 32.864585876464844,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013913258910179138,
+      "mask3_pct": 0.6018879413604736,
+      "neg_student_nll_loss": 0.014513559639453888,
+      "sampled_diff_pct": 0.607985258102417,
+      "step": 300
+    },
+    {
+      "adv1_neg_pct": 0.8508870601654053,
+      "adv2_active_pct": 0.859713077545166,
+      "ce_loss": 0.019685909152030945,
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 86.6822,
+      "eval_generator_rouge1": 99.4006,
+      "eval_generator_rouge2": 99.0209,
+      "eval_generator_rougeL": 98.951,
+      "eval_loss": 0.010082065127789974,
+      "eval_runtime": 595.0313,
+      "eval_samples_per_second": 5.738,
+      "eval_selector_accuracy": 93.9836,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.18,
+      "hpd_loss": 0.02267773076891899,
+      "k1_gt_raw_max": 0.14172008633613586,
+      "k1_gt_raw_mean": 0.0054550133645534515,
+      "k1_gt_raw_min": -0.005414052866399288,
+      "k1_gt_raw_neg_pct": 0.8508870601654053,
+      "k1_sample_max": -1.4249026775360107,
+      "k1_sample_mean": -1.6359399557113647,
+      "k1_sample_min": -1.8393499851226807,
+      "k1_sample_neg_pct": 36.4485969543457,
+      "mask3_pct": 0.859713077545166,
+      "neg_student_nll_loss": 0.019685909152030945,
+      "sampled_diff_pct": 0.859713077545166,
+      "step": 314
+    },
+    {
+      "adv1_neg_pct": 0.6673219203948975,
+      "adv2_active_pct": 1.2173185348510742,
+      "ce_loss": 0.011882005259394646,
+      "epoch": 1.0191463901076985,
+      "grad_norm": 4.386560916900635,
+      "hpd_loss": -0.0019697409588843584,
+      "k1_gt_raw_max": 0.14985336363315582,
+      "k1_gt_raw_mean": 0.006570600904524326,
+      "k1_gt_raw_min": -0.0011412458261474967,
+      "k1_gt_raw_neg_pct": 0.6673219203948975,
+      "k1_sample_max": -0.5733932256698608,
+      "k1_sample_mean": -0.7650949358940125,
+      "k1_sample_min": -0.9706472158432007,
+      "k1_sample_neg_pct": 36.458335876464844,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014574740827083588,
+      "mask3_pct": 1.2173185348510742,
+      "neg_student_nll_loss": 0.011882005259394646,
+      "sampled_diff_pct": 1.2173185348510742,
+      "step": 320
+    },
+    {
+      "adv1_neg_pct": 1.1466679573059082,
+      "adv2_active_pct": 0.4476942718029022,
+      "ce_loss": 0.009723825380206108,
+      "epoch": 1.0829676904666932,
+      "grad_norm": 1.2898244857788086,
+      "hpd_loss": 0.009684150107204914,
+      "k1_gt_raw_max": 0.10980867594480515,
+      "k1_gt_raw_mean": 0.0032046493142843246,
+      "k1_gt_raw_min": -0.009925473481416702,
+      "k1_gt_raw_neg_pct": 1.1466679573059082,
+      "k1_sample_max": -0.76816326379776,
+      "k1_sample_mean": -0.8626709580421448,
+      "k1_sample_min": -0.9573334455490112,
+      "k1_sample_neg_pct": 28.95833396911621,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.0096841499209404,
+      "mask3_pct": 0.4476942718029022,
+      "neg_student_nll_loss": 0.009723825380206108,
+      "sampled_diff_pct": 0.4520599842071533,
+      "step": 340
+    },
+    {
+      "adv1_neg_pct": 1.657383918762207,
+      "adv2_active_pct": 0.7416136860847473,
+      "ce_loss": 0.01363883912563324,
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.8189982175827026,
+      "hpd_loss": 0.012504098936915398,
+      "k1_gt_raw_max": 0.11255955696105957,
+      "k1_gt_raw_mean": 0.0030380175448954105,
+      "k1_gt_raw_min": -0.01135759986937046,
+      "k1_gt_raw_neg_pct": 1.657383918762207,
+      "k1_sample_max": -1.1351618766784668,
+      "k1_sample_mean": -1.3799619674682617,
+      "k1_sample_min": -1.6486986875534058,
+      "k1_sample_neg_pct": 30.0,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.012504099309444428,
+      "mask3_pct": 0.7416136860847473,
+      "neg_student_nll_loss": 0.01363883912563324,
+      "sampled_diff_pct": 0.7438143491744995,
+      "step": 360
+    },
+    {
+      "adv1_neg_pct": 1.3510011434555054,
+      "adv2_active_pct": 0.3445201814174652,
+      "ce_loss": 0.007902171462774277,
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.5892553329467773,
+      "hpd_loss": 0.008744428865611553,
+      "k1_gt_raw_max": 0.09419713914394379,
+      "k1_gt_raw_mean": 0.0022562379017472267,
+      "k1_gt_raw_min": -0.006403174251317978,
+      "k1_gt_raw_neg_pct": 1.3510011434555054,
+      "k1_sample_max": -0.583869457244873,
+      "k1_sample_mean": -0.7341683506965637,
+      "k1_sample_min": -0.8994753360748291,
+      "k1_sample_neg_pct": 23.75,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.008744429051876067,
+      "mask3_pct": 0.3445201814174652,
+      "neg_student_nll_loss": 0.007902171462774277,
+      "sampled_diff_pct": 0.35682976245880127,
+      "step": 380
+    },
+    {
+      "adv1_neg_pct": 1.4141113758087158,
+      "adv2_active_pct": 0.3374101221561432,
+      "ce_loss": 0.008193380199372768,
+      "epoch": 1.2744315915436777,
+      "grad_norm": 1.2929012775421143,
+      "hpd_loss": 0.006689602509140968,
+      "k1_gt_raw_max": 0.10671139508485794,
+      "k1_gt_raw_mean": 0.002954404801130295,
+      "k1_gt_raw_min": -0.006222248077392578,
+      "k1_gt_raw_neg_pct": 1.4141114950180054,
+      "k1_sample_max": -0.6661227941513062,
+      "k1_sample_mean": -0.7585095763206482,
+      "k1_sample_min": -0.8607568740844727,
+      "k1_sample_neg_pct": 25.760417938232422,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.0066896028816699985,
+      "mask3_pct": 0.3344895839691162,
+      "neg_student_nll_loss": 0.008193380199372768,
+      "sampled_diff_pct": 0.3449546694755554,
+      "step": 400
+    },
+    {
+      "adv1_neg_pct": 2.1220993995666504,
+      "adv2_active_pct": 0.4313049912452698,
+      "ce_loss": 0.011112527921795845,
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.8164143562316895,
+      "hpd_loss": 0.010808300226926804,
+      "k1_gt_raw_max": 0.10569541901350021,
+      "k1_gt_raw_mean": 0.0035689054057002068,
+      "k1_gt_raw_min": -0.004470471758395433,
+      "k1_gt_raw_neg_pct": 2.1220993995666504,
+      "k1_sample_max": -0.6986867189407349,
+      "k1_sample_mean": -0.7543543577194214,
+      "k1_sample_min": -0.8270018100738525,
+      "k1_sample_neg_pct": 25.279605865478516,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.010808297991752624,
+      "mask3_pct": 0.4313049912452698,
+      "neg_student_nll_loss": 0.011112527921795845,
+      "sampled_diff_pct": 0.4342671036720276,
+      "step": 420
+    },
+    {
+      "adv1_neg_pct": 1.9999275207519531,
+      "adv2_active_pct": 0.2967550456523895,
+      "ce_loss": 0.007173327729105949,
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.959374189376831,
+      "hpd_loss": 0.005844146944582462,
+      "k1_gt_raw_max": 0.08694229274988174,
+      "k1_gt_raw_mean": 0.002139701507985592,
+      "k1_gt_raw_min": -0.005928537808358669,
+      "k1_gt_raw_neg_pct": 1.9999275207519531,
+      "k1_sample_max": -0.8034830093383789,
+      "k1_sample_mean": -0.8980427980422974,
+      "k1_sample_min": -0.9926026463508606,
+      "k1_sample_neg_pct": 20.78125,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.005844147875905037,
+      "mask3_pct": 0.2967550456523895,
+      "neg_student_nll_loss": 0.007173327729105949,
+      "sampled_diff_pct": 0.30670204758644104,
+      "step": 440
+    },
+    {
+      "adv1_neg_pct": 1.9445667266845703,
+      "adv2_active_pct": 0.28356558084487915,
+      "ce_loss": 0.007787229493260384,
+      "epoch": 1.4658954926206622,
+      "grad_norm": 1.3586090803146362,
+      "hpd_loss": 0.009073459543287754,
+      "k1_gt_raw_max": 0.07425311952829361,
+      "k1_gt_raw_mean": 0.0016224482096731663,
+      "k1_gt_raw_min": -0.0022226697765290737,
+      "k1_gt_raw_neg_pct": 1.9445667266845703,
+      "k1_sample_max": -0.6454492807388306,
+      "k1_sample_mean": -0.686149001121521,
+      "k1_sample_min": -0.7300035953521729,
+      "k1_sample_neg_pct": 17.1875,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.009073460102081298,
+      "mask3_pct": 0.27911925315856934,
+      "neg_student_nll_loss": 0.007787229493260384,
+      "sampled_diff_pct": 0.28356558084487915,
+      "step": 460
+    },
+    {
+      "adv1_neg_pct": 1.8402546644210815,
+      "adv2_active_pct": 0.41872814297676086,
+      "ce_loss": 0.005175907164812088,
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.2023886740207672,
+      "hpd_loss": 0.0015706174308434129,
+      "k1_gt_raw_max": 0.07684825360774994,
+      "k1_gt_raw_mean": 0.0022208630107343197,
+      "k1_gt_raw_min": -0.008054896257817745,
+      "k1_gt_raw_neg_pct": 1.8402546644210815,
+      "k1_sample_max": -0.47196078300476074,
+      "k1_sample_mean": -0.4973880648612976,
+      "k1_sample_min": -0.5228154063224792,
+      "k1_sample_neg_pct": 17.5,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0015706172212958335,
+      "mask3_pct": 0.41872814297676086,
+      "neg_student_nll_loss": 0.005175907164812088,
+      "sampled_diff_pct": 0.4208254814147949,
+      "step": 480
+    },
+    {
+      "adv1_neg_pct": 2.5322799682617188,
+      "adv2_active_pct": 0.33805567026138306,
+      "ce_loss": 0.00585869699716568,
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.030877001583576202,
+      "hpd_loss": 0.003286010818555951,
+      "k1_gt_raw_max": 0.058261577039957047,
+      "k1_gt_raw_mean": 0.002064550295472145,
+      "k1_gt_raw_min": -0.0029995564837008715,
+      "k1_gt_raw_neg_pct": 2.5322799682617188,
+      "k1_sample_max": -0.3366956412792206,
+      "k1_sample_mean": -0.39731863141059875,
+      "k1_sample_min": -0.45794159173965454,
+      "k1_sample_neg_pct": 14.0625,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.0032860103994607924,
+      "mask3_pct": 0.33805567026138306,
+      "neg_student_nll_loss": 0.00585869699716568,
+      "sampled_diff_pct": 0.3419136703014374,
+      "step": 500
+    },
+    {
+      "adv1_neg_pct": 2.701911449432373,
+      "adv2_active_pct": 0.3643760681152344,
+      "ce_loss": 0.010007714852690697,
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.14292795956134796,
+      "hpd_loss": 0.012855440378189087,
+      "k1_gt_raw_max": 0.06845426559448242,
+      "k1_gt_raw_mean": 0.0017418819479644299,
+      "k1_gt_raw_min": -0.004030767362564802,
+      "k1_gt_raw_neg_pct": 2.701911449432373,
+      "k1_sample_max": -0.4802917242050171,
+      "k1_sample_mean": -0.5421380996704102,
+      "k1_sample_min": -0.6135058999061584,
+      "k1_sample_neg_pct": 17.03125,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.012855446338653565,
+      "mask3_pct": 0.3643760681152344,
+      "neg_student_nll_loss": 0.010007714852690697,
+      "sampled_diff_pct": 0.3710411787033081,
+      "step": 520
+    },
+    {
+      "adv1_neg_pct": 2.248600959777832,
+      "adv2_active_pct": 0.5108850598335266,
+      "ce_loss": 0.01003289595246315,
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.6681362390518188,
+      "hpd_loss": 0.009245548397302628,
+      "k1_gt_raw_max": 0.08858497440814972,
+      "k1_gt_raw_mean": 0.002853624988347292,
+      "k1_gt_raw_min": -0.004460392985492945,
+      "k1_gt_raw_neg_pct": 2.248600959777832,
+      "k1_sample_max": -0.7408795356750488,
+      "k1_sample_mean": -0.8169400691986084,
+      "k1_sample_min": -0.8906024694442749,
+      "k1_sample_neg_pct": 22.65625,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.009245550632476807,
+      "mask3_pct": 0.5108850598335266,
+      "neg_student_nll_loss": 0.01003289595246315,
+      "sampled_diff_pct": 0.5210334062576294,
+      "step": 540
+    },
+    {
+      "adv1_neg_pct": 2.4635496139526367,
+      "adv2_active_pct": 0.22115065157413483,
+      "ce_loss": 0.005004607141017914,
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.08243642747402191,
+      "hpd_loss": 0.0062531642615795135,
+      "k1_gt_raw_max": 0.0580272376537323,
+      "k1_gt_raw_mean": 0.0011763360816985369,
+      "k1_gt_raw_min": -0.005644598975777626,
+      "k1_gt_raw_neg_pct": 2.4635493755340576,
+      "k1_sample_max": -0.2404015064239502,
+      "k1_sample_mean": -0.2807300090789795,
+      "k1_sample_min": -0.32241004705429077,
+      "k1_sample_neg_pct": 14.921875,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.006253162026405334,
+      "mask3_pct": 0.22115065157413483,
+      "neg_student_nll_loss": 0.005004607141017914,
+      "sampled_diff_pct": 0.22733174264431,
+      "step": 560
+    },
+    {
+      "adv1_neg_pct": 3.3381173610687256,
+      "adv2_active_pct": 0.23258033394813538,
+      "ce_loss": 0.006839662324637175,
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.0869053527712822,
+      "hpd_loss": 0.00969111267477274,
+      "k1_gt_raw_max": 0.05798381567001343,
+      "k1_gt_raw_mean": 0.0016341875307261944,
+      "k1_gt_raw_min": -0.009443331509828568,
+      "k1_gt_raw_neg_pct": 3.3381173610687256,
+      "k1_sample_max": -0.29867684841156006,
+      "k1_sample_mean": -0.31859445571899414,
+      "k1_sample_min": -0.338512122631073,
+      "k1_sample_neg_pct": 10.625,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.009691111743450165,
+      "mask3_pct": 0.23258033394813538,
+      "neg_student_nll_loss": 0.006839662324637175,
+      "sampled_diff_pct": 0.24274389445781708,
+      "step": 580
+    },
+    {
+      "adv1_neg_pct": 4.040760040283203,
+      "adv2_active_pct": 0.20796719193458557,
+      "ce_loss": 0.002632163930684328,
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2512037456035614,
+      "hpd_loss": 0.0005975529784336686,
+      "k1_gt_raw_max": 0.049394719302654266,
+      "k1_gt_raw_mean": 0.0014062562258914113,
+      "k1_gt_raw_min": -0.002836518455296755,
+      "k1_gt_raw_neg_pct": 4.040760040283203,
+      "k1_sample_max": -0.23663434386253357,
+      "k1_sample_mean": -0.255886435508728,
+      "k1_sample_min": -0.2802615463733673,
+      "k1_sample_neg_pct": 10.208333969116211,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.0005975536536425352,
+      "mask3_pct": 0.20796719193458557,
+      "neg_student_nll_loss": 0.002632163930684328,
+      "sampled_diff_pct": 0.21867018938064575,
+      "step": 600
+    },
+    {
+      "adv1_neg_pct": 4.097904205322266,
+      "adv2_active_pct": 0.1848699152469635,
+      "ce_loss": 0.0032025063410401344,
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.338096022605896,
+      "hpd_loss": 0.002149906475096941,
+      "k1_gt_raw_max": 0.043635837733745575,
+      "k1_gt_raw_mean": 0.0007720005232840776,
+      "k1_gt_raw_min": -0.0036990540102124214,
+      "k1_gt_raw_neg_pct": 4.097904205322266,
+      "k1_sample_max": -0.2395995557308197,
+      "k1_sample_mean": -0.2737259864807129,
+      "k1_sample_min": -0.3078524172306061,
+      "k1_sample_neg_pct": 13.4375,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.0021499061957001688,
+      "mask3_pct": 0.1848699152469635,
+      "neg_student_nll_loss": 0.0032025063410401344,
+      "sampled_diff_pct": 0.18927133083343506,
+      "step": 620
+    },
+    {
+      "adv1_neg_pct": 3.390739917755127,
+      "adv2_active_pct": 0.10791555047035217,
+      "ce_loss": 0.001353576430119574,
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.1519,
+      "eval_generator_rouge1": 99.7488,
+      "eval_generator_rouge2": 99.6229,
+      "eval_generator_rougeL": 99.5782,
+      "eval_loss": 0.004672760609537363,
+      "eval_runtime": 596.1717,
+      "eval_samples_per_second": 5.727,
+      "eval_selector_accuracy": 98.014,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.179,
+      "hpd_loss": 0.0011793917510658503,
+      "k1_gt_raw_max": 0.03581507131457329,
+      "k1_gt_raw_mean": 0.0006900117732584476,
+      "k1_gt_raw_min": -0.002846459159627557,
+      "k1_gt_raw_neg_pct": 3.390739917755127,
+      "k1_sample_max": -0.06917751580476761,
+      "k1_sample_mean": -0.06961424648761749,
+      "k1_sample_min": -0.07005097717046738,
+      "k1_sample_neg_pct": 10.1694917678833,
+      "mask3_pct": 0.10791555047035217,
+      "neg_student_nll_loss": 0.001353576430119574,
+      "sampled_diff_pct": 0.10791555047035217,
+      "step": 628
+    },
+    {
+      "adv1_neg_pct": 3.476264715194702,
+      "adv2_active_pct": 0.12990054488182068,
+      "ce_loss": 0.004336793441325426,
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.6891085505485535,
+      "hpd_loss": 0.002609923481941223,
+      "k1_gt_raw_max": 0.038067810237407684,
+      "k1_gt_raw_mean": 0.0007896563620306551,
+      "k1_gt_raw_min": -0.005201837047934532,
+      "k1_gt_raw_neg_pct": 3.476264715194702,
+      "k1_sample_max": -0.2407418042421341,
+      "k1_sample_mean": -0.2449159324169159,
+      "k1_sample_min": -0.2522057890892029,
+      "k1_sample_neg_pct": 9.798177719116211,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.0020211638882756235,
+      "mask3_pct": 0.12990054488182068,
+      "neg_student_nll_loss": 0.004336793441325426,
+      "sampled_diff_pct": 0.14956170320510864,
+      "step": 640
+    },
+    {
+      "adv1_neg_pct": 3.8216357231140137,
+      "adv2_active_pct": 0.16393554210662842,
+      "ce_loss": 0.0034592957235872746,
+      "epoch": 2.102114080574392,
+      "grad_norm": 1.311052918434143,
+      "hpd_loss": 0.004151628818362951,
+      "k1_gt_raw_max": 0.04581638425588608,
+      "k1_gt_raw_mean": 0.0008853404433466494,
+      "k1_gt_raw_min": -0.0063157654367387295,
+      "k1_gt_raw_neg_pct": 3.8216357231140137,
+      "k1_sample_max": -0.250868022441864,
+      "k1_sample_mean": -0.30226844549179077,
+      "k1_sample_min": -0.3555181622505188,
+      "k1_sample_neg_pct": 10.833333969116211,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.004151628911495208,
+      "mask3_pct": 0.16393554210662842,
+      "neg_student_nll_loss": 0.0034592957235872746,
+      "sampled_diff_pct": 0.16612085700035095,
+      "step": 660
+    },
+    {
+      "adv1_neg_pct": 4.672894477844238,
+      "adv2_active_pct": 0.17785239219665527,
+      "ce_loss": 0.0037891073152422905,
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.20577986538410187,
+      "hpd_loss": 0.004049420822411776,
+      "k1_gt_raw_max": 0.047759026288986206,
+      "k1_gt_raw_mean": 0.0010929291602224112,
+      "k1_gt_raw_min": -0.008511744439601898,
+      "k1_gt_raw_neg_pct": 4.672894477844238,
+      "k1_sample_max": -0.2957162857055664,
+      "k1_sample_mean": -0.3683391511440277,
+      "k1_sample_min": -0.440962016582489,
+      "k1_sample_neg_pct": 12.8125,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.004049421846866607,
+      "mask3_pct": 0.17785239219665527,
+      "neg_student_nll_loss": 0.0037891073152422905,
+      "sampled_diff_pct": 0.17980551719665527,
+      "step": 680
+    },
+    {
+      "adv1_neg_pct": 3.504452705383301,
+      "adv2_active_pct": 0.33364731073379517,
+      "ce_loss": 0.00837249867618084,
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.05891747772693634,
+      "hpd_loss": 0.012418361380696297,
+      "k1_gt_raw_max": 0.044552117586135864,
+      "k1_gt_raw_mean": 0.001460403436794877,
+      "k1_gt_raw_min": -0.0013761408627033234,
+      "k1_gt_raw_neg_pct": 3.504452705383301,
+      "k1_sample_max": -0.41403040289878845,
+      "k1_sample_mean": -0.4155550003051758,
+      "k1_sample_min": -0.4170795679092407,
+      "k1_sample_neg_pct": 12.8125,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.012418360263109208,
+      "mask3_pct": 0.33364731073379517,
+      "neg_student_nll_loss": 0.00837249867618084,
+      "sampled_diff_pct": 0.33364731073379517,
+      "step": 700
+    },
+    {
+      "adv1_neg_pct": 2.9557998180389404,
+      "adv2_active_pct": 0.2356344759464264,
+      "ce_loss": 0.002732612658292055,
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.03069828823208809,
+      "hpd_loss": 0.0013308569323271513,
+      "k1_gt_raw_max": 0.043037086725234985,
+      "k1_gt_raw_mean": 0.001189029891975224,
+      "k1_gt_raw_min": -0.00335265276953578,
+      "k1_gt_raw_neg_pct": 2.9557998180389404,
+      "k1_sample_max": -0.12545080482959747,
+      "k1_sample_mean": -0.13167737424373627,
+      "k1_sample_min": -0.13885264098644257,
+      "k1_sample_neg_pct": 12.1875,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.0013308576308190823,
+      "mask3_pct": 0.2356344759464264,
+      "neg_student_nll_loss": 0.002732612658292055,
+      "sampled_diff_pct": 0.24056380987167358,
+      "step": 720
+    },
+    {
+      "adv1_neg_pct": 4.362781524658203,
+      "adv2_active_pct": 0.14987114071846008,
+      "ce_loss": 0.00366773409768939,
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.03650576248764992,
+      "hpd_loss": 0.0028897470328956842,
+      "k1_gt_raw_max": 0.03678428381681442,
+      "k1_gt_raw_mean": 0.000713786983396858,
+      "k1_gt_raw_min": -0.007670572958886623,
+      "k1_gt_raw_neg_pct": 4.362782001495361,
+      "k1_sample_max": -0.16289430856704712,
+      "k1_sample_mean": -0.19424015283584595,
+      "k1_sample_min": -0.2255859375,
+      "k1_sample_neg_pct": 9.0625,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.002889746427536011,
+      "mask3_pct": 0.14987114071846008,
+      "neg_student_nll_loss": 0.00366773409768939,
+      "sampled_diff_pct": 0.1667182892560959,
+      "step": 740
+    },
+    {
+      "adv1_neg_pct": 4.3088059425354,
+      "adv2_active_pct": 0.073929063975811,
+      "ce_loss": 0.0018987958319485188,
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.03706243261694908,
+      "hpd_loss": 0.001820039818994701,
+      "k1_gt_raw_max": 0.030663518235087395,
+      "k1_gt_raw_mean": 0.0005116415559314191,
+      "k1_gt_raw_min": -0.003718154039233923,
+      "k1_gt_raw_neg_pct": 4.3088059425354,
+      "k1_sample_max": -0.11870913207530975,
+      "k1_sample_mean": -0.12011738121509552,
+      "k1_sample_min": -0.12152564525604248,
+      "k1_sample_neg_pct": 7.03125,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.0018200401216745376,
+      "mask3_pct": 0.073929063975811,
+      "neg_student_nll_loss": 0.0018987958319485188,
+      "sampled_diff_pct": 0.08056812733411789,
+      "step": 760
+    },
+    {
+      "adv1_neg_pct": 4.6282758712768555,
+      "adv2_active_pct": 0.08583934605121613,
+      "ce_loss": 0.000978614203631878,
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.0633927583694458,
+      "hpd_loss": 0.0008434242918156087,
+      "k1_gt_raw_max": 0.02713455632328987,
+      "k1_gt_raw_mean": 0.0004272414662409574,
+      "k1_gt_raw_min": -0.006445502862334251,
+      "k1_gt_raw_neg_pct": 4.628275394439697,
+      "k1_sample_max": -0.10302496701478958,
+      "k1_sample_mean": -0.11016625910997391,
+      "k1_sample_min": -0.11730755120515823,
+      "k1_sample_neg_pct": 8.90625,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0008434242568910122,
+      "mask3_pct": 0.08583934605121613,
+      "neg_student_nll_loss": 0.000978614203631878,
+      "sampled_diff_pct": 0.09829449653625488,
+      "step": 780
+    },
+    {
+      "adv1_neg_pct": 5.204377174377441,
+      "adv2_active_pct": 0.15303727984428406,
+      "ce_loss": 0.004278196021914482,
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.2113279402256012,
+      "hpd_loss": 0.00520919868722558,
+      "k1_gt_raw_max": 0.0305880606174469,
+      "k1_gt_raw_mean": 0.0005127020995132625,
+      "k1_gt_raw_min": -0.0020859134383499622,
+      "k1_gt_raw_neg_pct": 5.204377174377441,
+      "k1_sample_max": -0.07243171334266663,
+      "k1_sample_mean": -0.08607004582881927,
+      "k1_sample_min": -0.10009756684303284,
+      "k1_sample_neg_pct": 10.0,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.005209198221564293,
+      "mask3_pct": 0.15303727984428406,
+      "neg_student_nll_loss": 0.004278196021914482,
+      "sampled_diff_pct": 0.1585448682308197,
+      "step": 800
+    },
+    {
+      "adv1_neg_pct": 5.268265724182129,
+      "adv2_active_pct": 0.10696102678775787,
+      "ce_loss": 0.003248082473874092,
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.32251229882240295,
+      "hpd_loss": 0.002392602153122425,
+      "k1_gt_raw_max": 0.03637195751070976,
+      "k1_gt_raw_mean": 0.0010833492269739509,
+      "k1_gt_raw_min": -0.0027411947958171368,
+      "k1_gt_raw_neg_pct": 5.268265724182129,
+      "k1_sample_max": -0.06464927643537521,
+      "k1_sample_mean": -0.06609892845153809,
+      "k1_sample_min": -0.06948131322860718,
+      "k1_sample_neg_pct": 6.826923370361328,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.0023926014080643656,
+      "mask3_pct": 0.1040404736995697,
+      "neg_student_nll_loss": 0.003248082473874092,
+      "sampled_diff_pct": 0.11661312729120255,
+      "step": 820
+    },
+    {
+      "adv1_neg_pct": 5.766000270843506,
+      "adv2_active_pct": 0.06631548702716827,
+      "ce_loss": 0.0020227693021297455,
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.2545914947986603,
+      "hpd_loss": 0.002442866563796997,
+      "k1_gt_raw_max": 0.027828488498926163,
+      "k1_gt_raw_mean": 0.0007106282282620668,
+      "k1_gt_raw_min": -0.004438548348844051,
+      "k1_gt_raw_neg_pct": 5.766000270843506,
+      "k1_sample_max": -0.10070931911468506,
+      "k1_sample_mean": -0.10070931911468506,
+      "k1_sample_min": -0.10070931911468506,
+      "k1_sample_neg_pct": 7.1875,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.002442866563796997,
+      "mask3_pct": 0.06631548702716827,
+      "neg_student_nll_loss": 0.0020227693021297455,
+      "sampled_diff_pct": 0.07493838667869568,
+      "step": 840
+    },
+    {
+      "adv1_neg_pct": 6.06646728515625,
+      "adv2_active_pct": 0.10010633617639542,
+      "ce_loss": 0.0019979095086455345,
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.2798786759376526,
+      "hpd_loss": 0.0019781002774834633,
+      "k1_gt_raw_max": 0.0327674075961113,
+      "k1_gt_raw_mean": 0.0005696950247511268,
+      "k1_gt_raw_min": -0.003440997563302517,
+      "k1_gt_raw_neg_pct": 6.06646728515625,
+      "k1_sample_max": -0.08324240148067474,
+      "k1_sample_mean": -0.13795003294944763,
+      "k1_sample_min": -0.1926576793193817,
+      "k1_sample_neg_pct": 9.375,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.0019781002774834633,
+      "mask3_pct": 0.10010633617639542,
+      "neg_student_nll_loss": 0.0019979095086455345,
+      "sampled_diff_pct": 0.10264699161052704,
+      "step": 860
+    },
+    {
+      "adv1_neg_pct": 5.891697883605957,
+      "adv2_active_pct": 0.07234375178813934,
+      "ce_loss": 0.0014330728445202112,
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.09429220855236053,
+      "hpd_loss": 0.0010787295177578926,
+      "k1_gt_raw_max": 0.038590773940086365,
+      "k1_gt_raw_mean": 0.0007731987861916423,
+      "k1_gt_raw_min": -0.0038950724992901087,
+      "k1_gt_raw_neg_pct": 5.891697883605957,
+      "k1_sample_max": -0.1224956065416336,
+      "k1_sample_mean": -0.1250840276479721,
+      "k1_sample_min": -0.12767243385314941,
+      "k1_sample_neg_pct": 7.34375,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.001078729424625635,
+      "mask3_pct": 0.07234375178813934,
+      "neg_student_nll_loss": 0.0014330728445202112,
+      "sampled_diff_pct": 0.07657574117183685,
+      "step": 880
+    },
+    {
+      "adv1_neg_pct": 6.011615753173828,
+      "adv2_active_pct": 0.09838910400867462,
+      "ce_loss": 0.0020957919768989086,
+      "epoch": 2.8679696848823295,
+      "grad_norm": 1.0214177370071411,
+      "hpd_loss": 0.0015987120568752289,
+      "k1_gt_raw_max": 0.03688127547502518,
+      "k1_gt_raw_mean": 0.0006160076009109616,
+      "k1_gt_raw_min": -0.005018594674766064,
+      "k1_gt_raw_neg_pct": 6.011615753173828,
+      "k1_sample_max": -0.2794632613658905,
+      "k1_sample_mean": -0.2854064106941223,
+      "k1_sample_min": -0.2913495600223541,
+      "k1_sample_neg_pct": 8.75,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.0015987120568752289,
+      "mask3_pct": 0.09838910400867462,
+      "neg_student_nll_loss": 0.0020957919768989086,
+      "sampled_diff_pct": 0.10469574481248856,
+      "step": 900
+    },
+    {
+      "adv1_neg_pct": 5.8451714515686035,
+      "adv2_active_pct": 0.12536077201366425,
+      "ce_loss": 0.0031956052407622337,
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.009789629839360714,
+      "hpd_loss": 0.005076252389699221,
+      "k1_gt_raw_max": 0.03244764730334282,
+      "k1_gt_raw_mean": 0.0005789952119812369,
+      "k1_gt_raw_min": -0.004300437867641449,
+      "k1_gt_raw_neg_pct": 5.8451714515686035,
+      "k1_sample_max": -0.09452944993972778,
+      "k1_sample_mean": -0.1174035519361496,
+      "k1_sample_min": -0.1418294906616211,
+      "k1_sample_neg_pct": 8.4375,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.005076252669095993,
+      "mask3_pct": 0.12536077201366425,
+      "neg_student_nll_loss": 0.0031956052407622337,
+      "sampled_diff_pct": 0.129419207572937,
+      "step": 920
+    },
+    {
+      "adv1_neg_pct": 6.187288284301758,
+      "adv2_active_pct": 0.08470146358013153,
+      "ce_loss": 0.001648780656978488,
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.1643683910369873,
+      "hpd_loss": 0.0010852484265342355,
+      "k1_gt_raw_max": 0.03225347399711609,
+      "k1_gt_raw_mean": 0.0005872219335287809,
+      "k1_gt_raw_min": -0.0068657128140330315,
+      "k1_gt_raw_neg_pct": 6.187288284301758,
+      "k1_sample_max": -0.11387737095355988,
+      "k1_sample_mean": -0.13910287618637085,
+      "k1_sample_min": -0.1643284261226654,
+      "k1_sample_neg_pct": 7.8125,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.0010852484032511712,
+      "mask3_pct": 0.08470146358013153,
+      "neg_student_nll_loss": 0.001648780656978488,
+      "sampled_diff_pct": 0.08656158298254013,
+      "step": 940
+    },
+    {
+      "adv1_neg_pct": 5.000968933105469,
+      "adv2_active_pct": 0.110027015209198,
+      "ce_loss": 0.001415811711922288,
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.6776,
+      "eval_generator_rouge1": 99.8328,
+      "eval_generator_rouge2": 99.7335,
+      "eval_generator_rougeL": 99.6469,
+      "eval_loss": 0.00400863541290164,
+      "eval_runtime": 591.8273,
+      "eval_samples_per_second": 5.769,
+      "eval_selector_accuracy": 97.7804,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.181,
+      "hpd_loss": 0.0011558050755411386,
+      "k1_gt_raw_max": 0.03289224952459335,
+      "k1_gt_raw_mean": 0.0007148425211198628,
+      "k1_gt_raw_min": -0.0005029054591432214,
+      "k1_gt_raw_neg_pct": 5.000969409942627,
+      "k1_sample_max": -0.4340977072715759,
+      "k1_sample_mean": -0.4340977072715759,
+      "k1_sample_min": -0.4340977072715759,
+      "k1_sample_neg_pct": 9.090909004211426,
+      "mask3_pct": 0.110027015209198,
+      "neg_student_nll_loss": 0.001415811711922288,
+      "sampled_diff_pct": 0.110027015209198,
+      "step": 942
+    },
+    {
+      "adv1_neg_pct": 5.784729957580566,
+      "adv2_active_pct": 0.1198313981294632,
+      "ce_loss": 0.0024845735169947147,
+      "epoch": 3.057439170323095,
+      "grad_norm": 0.0822562500834465,
+      "hpd_loss": 0.0026429281570017338,
+      "k1_gt_raw_max": 0.030722517520189285,
+      "k1_gt_raw_mean": 0.0009094014531001449,
+      "k1_gt_raw_min": -0.006959723308682442,
+      "k1_gt_raw_neg_pct": 5.784729957580566,
+      "k1_sample_max": -0.06599755585193634,
+      "k1_sample_mean": -0.06599755585193634,
+      "k1_sample_min": -0.06599755585193634,
+      "k1_sample_neg_pct": 6.94444465637207,
+      "learning_rate": 3.3007844105971336e-05,
+      "loss": 0.0024656012654304506,
+      "mask3_pct": 0.1198313981294632,
+      "neg_student_nll_loss": 0.0024845735169947147,
+      "sampled_diff_pct": 0.12411808967590332,
+      "step": 960
+    },
+    {
+      "adv1_neg_pct": 6.275859832763672,
+      "adv2_active_pct": 0.1653088629245758,
+      "ce_loss": 0.002910045674070716,
+      "epoch": 3.12126047068209,
+      "grad_norm": 0.002782815834507346,
+      "hpd_loss": 0.002390575595200062,
+      "k1_gt_raw_max": 0.03876848146319389,
+      "k1_gt_raw_mean": 0.001082484028302133,
+      "k1_gt_raw_min": -0.005310694221407175,
+      "k1_gt_raw_neg_pct": 6.275859832763672,
+      "k1_sample_max": -0.10106666386127472,
+      "k1_sample_mean": -0.10889029502868652,
+      "k1_sample_min": -0.11623653769493103,
+      "k1_sample_neg_pct": 7.8125,
+      "learning_rate": 3.114292593987149e-05,
+      "loss": 0.0023905765265226364,
+      "mask3_pct": 0.1653088629245758,
+      "neg_student_nll_loss": 0.002910045674070716,
+      "sampled_diff_pct": 0.17157939076423645,
+      "step": 980
+    },
+    {
+      "adv1_neg_pct": 6.713335037231445,
+      "adv2_active_pct": 0.07989098131656647,
+      "ce_loss": 0.000890995841473341,
+      "epoch": 3.185081771041085,
+      "grad_norm": 0.025590306147933006,
+      "hpd_loss": 0.0010361053282395005,
+      "k1_gt_raw_max": 0.02309339866042137,
+      "k1_gt_raw_mean": 0.000387923966627568,
+      "k1_gt_raw_min": -0.0018807905726134777,
+      "k1_gt_raw_neg_pct": 6.713335037231445,
+      "k1_sample_max": -0.07003647089004517,
+      "k1_sample_mean": -0.0700383111834526,
+      "k1_sample_min": -0.07004015147686005,
+      "k1_sample_neg_pct": 6.5625,
+      "learning_rate": 2.9308285746383408e-05,
+      "loss": 0.0010361054912209511,
+      "mask3_pct": 0.07989098131656647,
+      "neg_student_nll_loss": 0.000890995841473341,
+      "sampled_diff_pct": 0.08394942432641983,
+      "step": 1000
+    },
+    {
+      "adv1_neg_pct": 6.980464458465576,
+      "adv2_active_pct": 0.04861818999052048,
+      "ce_loss": 0.0008955475641414523,
+      "epoch": 3.24890307140008,
+      "grad_norm": 0.04064895957708359,
+      "hpd_loss": 0.0005541563150472939,
+      "k1_gt_raw_max": 0.023353416472673416,
+      "k1_gt_raw_mean": 0.00037372263614088297,
+      "k1_gt_raw_min": -0.0031757489778101444,
+      "k1_gt_raw_neg_pct": 6.980464458465576,
+      "k1_sample_max": -0.02265203185379505,
+      "k1_sample_mean": -0.02265203185379505,
+      "k1_sample_min": -0.02265203185379505,
+      "k1_sample_neg_pct": 5.625,
+      "learning_rate": 2.7506861536187867e-05,
+      "loss": 0.0005541563965380192,
+      "mask3_pct": 0.04861818999052048,
+      "neg_student_nll_loss": 0.0008955475641414523,
+      "sampled_diff_pct": 0.04861818999052048,
+      "step": 1020
+    },
+    {
+      "adv1_neg_pct": 6.828683853149414,
+      "adv2_active_pct": 0.0889447033405304,
+      "ce_loss": 0.0019629367161542177,
+      "epoch": 3.3127243717590744,
+      "grad_norm": 2.8549108505249023,
+      "hpd_loss": 0.002801127266138792,
+      "k1_gt_raw_max": 0.021204540506005287,
+      "k1_gt_raw_mean": 0.00043139539775438607,
+      "k1_gt_raw_min": -0.006724623031914234,
+      "k1_gt_raw_neg_pct": 6.828684329986572,
+      "k1_sample_max": -0.07739919424057007,
+      "k1_sample_mean": -0.07739919424057007,
+      "k1_sample_min": -0.07739919424057007,
+      "k1_sample_neg_pct": 5.0,
+      "learning_rate": 2.5741538127566396e-05,
+      "loss": 0.00280112698674202,
+      "mask3_pct": 0.0889447033405304,
+      "neg_student_nll_loss": 0.0019629367161542177,
+      "sampled_diff_pct": 0.10269542038440704,
+      "step": 1040
+    },
+    {
+      "adv1_neg_pct": 7.2235517501831055,
+      "adv2_active_pct": 0.10061224550008774,
+      "ce_loss": 0.004100818186998367,
+      "epoch": 3.3765456721180693,
+      "grad_norm": 0.004515279550105333,
+      "hpd_loss": 0.006908780429512262,
+      "k1_gt_raw_max": 0.017903421074151993,
+      "k1_gt_raw_mean": 0.0002864316338673234,
+      "k1_gt_raw_min": -0.003864757716655731,
+      "k1_gt_raw_neg_pct": 7.2235517501831055,
+      "k1_sample_max": -0.07832760363817215,
+      "k1_sample_mean": -0.07832760363817215,
+      "k1_sample_min": -0.07832760363817215,
+      "k1_sample_neg_pct": 5.9375,
+      "learning_rate": 2.4015142526625115e-05,
+      "loss": 0.006908781081438065,
+      "mask3_pct": 0.10061224550008774,
+      "neg_student_nll_loss": 0.004100818186998367,
+      "sampled_diff_pct": 0.10612311214208603,
+      "step": 1060
+    },
+    {
+      "adv1_neg_pct": 7.121249198913574,
+      "adv2_active_pct": 0.035785384476184845,
+      "ce_loss": 0.0010942043736577034,
+      "epoch": 3.4403669724770642,
+      "grad_norm": 0.006863700225949287,
+      "hpd_loss": 0.0009617547038942575,
+      "k1_gt_raw_max": 0.016735289245843887,
+      "k1_gt_raw_mean": 0.0003524690982885659,
+      "k1_gt_raw_min": -0.003919572569429874,
+      "k1_gt_raw_neg_pct": 7.121249198913574,
+      "k1_sample_max": -0.02336699143052101,
+      "k1_sample_mean": -0.02336699143052101,
+      "k1_sample_min": -0.02336699143052101,
+      "k1_sample_neg_pct": 4.0625,
+      "learning_rate": 2.233043940009967e-05,
+      "loss": 0.0009617548435926438,
+      "mask3_pct": 0.032108910381793976,
+      "neg_student_nll_loss": 0.0010942043736577034,
+      "sampled_diff_pct": 0.05000416934490204,
+      "step": 1080
+    },
+    {
+      "adv1_neg_pct": 7.497800827026367,
+      "adv2_active_pct": 0.06837265938520432,
+      "ce_loss": 0.0012112363474443555,
+      "epoch": 3.504188272836059,
+      "grad_norm": 0.009582476690411568,
+      "hpd_loss": 0.0011069111060351133,
+      "k1_gt_raw_max": 0.01962250843644142,
+      "k1_gt_raw_mean": 0.0003861070144921541,
+      "k1_gt_raw_min": -0.003895876696333289,
+      "k1_gt_raw_neg_pct": 7.497800827026367,
+      "k1_sample_max": -0.03470407798886299,
+      "k1_sample_mean": -0.038440171629190445,
+      "k1_sample_min": -0.0421762689948082,
+      "k1_sample_neg_pct": 5.78125,
+      "learning_rate": 2.0690126647990974e-05,
+      "loss": 0.0011069110594689846,
+      "mask3_pct": 0.06837265938520432,
+      "neg_student_nll_loss": 0.0012112363474443555,
+      "sampled_diff_pct": 0.07695083320140839,
+      "step": 1100
+    },
+    {
+      "adv1_neg_pct": 7.4989423751831055,
+      "adv2_active_pct": 0.09932753443717957,
+      "ce_loss": 0.001159896608442068,
+      "epoch": 3.568009573195054,
+      "grad_norm": 0.07570958882570267,
+      "hpd_loss": 0.0004915354074910283,
+      "k1_gt_raw_max": 0.021993957459926605,
+      "k1_gt_raw_mean": 0.0004521916271187365,
+      "k1_gt_raw_min": -0.005119780544191599,
+      "k1_gt_raw_neg_pct": 7.4989423751831055,
+      "k1_sample_max": -0.03237403556704521,
+      "k1_sample_mean": -0.06356782466173172,
+      "k1_sample_min": -0.09476162493228912,
+      "k1_sample_neg_pct": 6.5625,
+      "learning_rate": 1.9096831083121828e-05,
+      "loss": 0.0004915353842079639,
+      "mask3_pct": 0.09932753443717957,
+      "neg_student_nll_loss": 0.001159896608442068,
+      "sampled_diff_pct": 0.10849244892597198,
+      "step": 1120
+    },
+    {
+      "adv1_neg_pct": 7.469758033752441,
+      "adv2_active_pct": 0.03868325054645538,
+      "ce_loss": 0.0013367629144340754,
+      "epoch": 3.6318308735540485,
+      "grad_norm": 0.030605977401137352,
+      "hpd_loss": 0.0013018195750191808,
+      "k1_gt_raw_max": 0.01742619276046753,
+      "k1_gt_raw_mean": 0.00042788474820554256,
+      "k1_gt_raw_min": -0.002206374891102314,
+      "k1_gt_raw_neg_pct": 7.469758033752441,
+      "k1_sample_max": -0.026432175189256668,
+      "k1_sample_mean": -0.026432175189256668,
+      "k1_sample_min": -0.026432175189256668,
+      "k1_sample_neg_pct": 4.375,
+      "learning_rate": 1.7553104224533324e-05,
+      "loss": 0.0013018193654716014,
+      "mask3_pct": 0.036279406398534775,
+      "neg_student_nll_loss": 0.0013367629144340754,
+      "sampled_diff_pct": 0.04094774276018143,
+      "step": 1140
+    },
+    {
+      "adv1_neg_pct": 7.294529914855957,
+      "adv2_active_pct": 0.11667037010192871,
+      "ce_loss": 0.0022262034472078085,
+      "epoch": 3.6956521739130435,
+      "grad_norm": 0.1906837671995163,
+      "hpd_loss": 0.0014206715859472752,
+      "k1_gt_raw_max": 0.021399978548288345,
+      "k1_gt_raw_mean": 0.00044373038690537214,
+      "k1_gt_raw_min": -0.0028102025389671326,
+      "k1_gt_raw_neg_pct": 7.294529914855957,
+      "k1_sample_max": -0.02788049727678299,
+      "k1_sample_mean": -0.028852656483650208,
+      "k1_sample_min": -0.03075457364320755,
+      "k1_sample_neg_pct": 4.9479169845581055,
+      "learning_rate": 1.606141821145735e-05,
+      "loss": 0.0014206715859472752,
+      "mask3_pct": 0.11374981701374054,
+      "neg_student_nll_loss": 0.0022262034472078085,
+      "sampled_diff_pct": 0.1271422654390335,
+      "step": 1160
+    },
+    {
+      "adv1_neg_pct": 7.579638957977295,
+      "adv2_active_pct": 0.06752925366163254,
+      "ce_loss": 0.0028973212465643883,
+      "epoch": 3.7594734742720384,
+      "grad_norm": 0.011799277737736702,
+      "hpd_loss": 0.0015698154456913471,
+      "k1_gt_raw_max": 0.01623934879899025,
+      "k1_gt_raw_mean": 0.00032681378070265055,
+      "k1_gt_raw_min": -0.0041321092285215855,
+      "k1_gt_raw_neg_pct": 7.579638957977295,
+      "k1_sample_max": -0.0820823684334755,
+      "k1_sample_mean": -0.08302680402994156,
+      "k1_sample_min": -0.08577128499746323,
+      "k1_sample_neg_pct": 3.680555582046509,
+      "learning_rate": 1.4624161844408763e-05,
+      "loss": 0.0015698155388236047,
+      "mask3_pct": 0.06483529508113861,
+      "neg_student_nll_loss": 0.0028973212465643883,
+      "sampled_diff_pct": 0.08027052134275436,
+      "step": 1180
+    },
+    {
+      "adv1_neg_pct": 8.004940032958984,
+      "adv2_active_pct": 0.17861586809158325,
+      "ce_loss": 0.0022735765669494867,
+      "epoch": 3.823294774631033,
+      "grad_norm": 0.014485353603959084,
+      "hpd_loss": 0.0013695353409275413,
+      "k1_gt_raw_max": 0.022271674126386642,
+      "k1_gt_raw_mean": 0.00047311594244092703,
+      "k1_gt_raw_min": -0.005409082397818565,
+      "k1_gt_raw_neg_pct": 8.004940032958984,
+      "k1_sample_max": -0.07869651913642883,
+      "k1_sample_mean": -0.1084669977426529,
+      "k1_sample_min": -0.13823749125003815,
+      "k1_sample_neg_pct": 6.5625,
+      "learning_rate": 1.324363675973698e-05,
+      "loss": 0.0013695357367396355,
+      "mask3_pct": 0.17861586809158325,
+      "neg_student_nll_loss": 0.0022735765669494867,
+      "sampled_diff_pct": 0.1825408637523651,
+      "step": 1200
+    },
+    {
+      "adv1_neg_pct": 7.9708051681518555,
+      "adv2_active_pct": 0.14446821808815002,
+      "ce_loss": 0.002031442243605852,
+      "epoch": 3.8871160749900278,
+      "grad_norm": 1.8391499519348145,
+      "hpd_loss": 0.0014964366564527154,
+      "k1_gt_raw_max": 0.019910190254449844,
+      "k1_gt_raw_mean": 0.0005614210967905819,
+      "k1_gt_raw_min": -0.003311981214210391,
+      "k1_gt_raw_neg_pct": 7.9708051681518555,
+      "k1_sample_max": -0.06566861271858215,
+      "k1_sample_mean": -0.06566861271858215,
+      "k1_sample_min": -0.06566861271858215,
+      "k1_sample_neg_pct": 5.3125,
+      "learning_rate": 1.1922053743763052e-05,
+      "loss": 0.0014964366331696511,
+      "mask3_pct": 0.14446821808815002,
+      "neg_student_nll_loss": 0.002031442243605852,
+      "sampled_diff_pct": 0.1486356258392334,
+      "step": 1220
+    },
+    {
+      "adv1_neg_pct": 7.752712249755859,
+      "adv2_active_pct": 0.05451521277427673,
+      "ce_loss": 0.0008983995649032295,
+      "epoch": 3.9509373753490227,
+      "grad_norm": 0.015798989683389664,
+      "hpd_loss": 0.0006963292835280299,
+      "k1_gt_raw_max": 0.017497122287750244,
+      "k1_gt_raw_mean": 0.00027755158953368664,
+      "k1_gt_raw_min": -0.0025472629349678755,
+      "k1_gt_raw_neg_pct": 7.752712249755859,
+      "k1_sample_max": -0.0381598025560379,
+      "k1_sample_mean": -0.03892132267355919,
+      "k1_sample_min": -0.03968284651637077,
+      "k1_sample_neg_pct": 4.375,
+      "learning_rate": 1.0661529192404973e-05,
+      "loss": 0.0006963292602449655,
+      "mask3_pct": 0.05451521277427673,
+      "neg_student_nll_loss": 0.0008983995649032295,
+      "sampled_diff_pct": 0.061319805681705475,
+      "step": 1240
+    },
+    {
+      "adv1_neg_pct": 7.9339280128479,
+      "adv2_active_pct": 0.0740404948592186,
+      "ce_loss": 0.0014802285004407167,
+      "epoch": 4.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.4953,
+      "eval_generator_rouge1": 99.8312,
+      "eval_generator_rouge2": 99.7559,
+      "eval_generator_rougeL": 99.645,
+      "eval_loss": 0.003369994228705764,
+      "eval_runtime": 1103.3525,
+      "eval_samples_per_second": 3.094,
+      "eval_selector_accuracy": 98.5981,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.097,
+      "hpd_loss": 0.0012612877180799842,
+      "k1_gt_raw_max": 0.01940547674894333,
+      "k1_gt_raw_mean": 0.0002844607806764543,
+      "k1_gt_raw_min": -0.003394282190129161,
+      "k1_gt_raw_neg_pct": 7.9339280128479,
+      "k1_sample_max": -0.06970421969890594,
+      "k1_sample_mean": -0.06970421969890594,
+      "k1_sample_min": -0.06970421969890594,
+      "k1_sample_neg_pct": 6.910569190979004,
+      "mask3_pct": 0.0740404948592186,
+      "neg_student_nll_loss": 0.0014802285004407167,
+      "sampled_diff_pct": 0.08771276473999023,
+      "step": 1256
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.697643190059008e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/hpd/checkpoint-1256/training_args.bin b/qwen3/hpd/checkpoint-1256/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1caa0e72cc5edd590e105073725cf4d832684e44
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f04fac21a72f04221af020c3bb88a0dc0ccf53bb7a0c0da21d1ab444937cb9d
+size 7185
diff --git a/qwen3/hpd/checkpoint-1256/zero_to_fp32.py b/qwen3/hpd/checkpoint-1256/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1256/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/hpd/checkpoint-1570/README.md b/qwen3/hpd/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1570/adapter_config.json b/qwen3/hpd/checkpoint-1570/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0f6594b49d741b9fe4eb4a1a4df0b797e15bcd
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1570/adapter_model.safetensors b/qwen3/hpd/checkpoint-1570/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3a366629662cf5ee4469936aa378a672e2f2651e
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aaf8b6a7a4f0e6d4b7fbe24b411c1a83ffd5bd908302ea9c83a79f8b006567e
+size 40422560
diff --git a/qwen3/hpd/checkpoint-1570/chat_template.jinja b/qwen3/hpd/checkpoint-1570/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bae7d7aecc86c0762f0ffaf774d9ae038fd58b5
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60d32d12877c7ff9735a666636e0715c69a848b25e432ad7de37806a59a6a950
+size 121135557
diff --git a/qwen3/hpd/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46bbb085fab40fc5e1eb1ae6ecd6b846bd9c7a29
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:100873d96038aa0628c816d0caa942c962a5c6942215f4e0bc25b61a7a126cf5
+size 121135749
diff --git a/qwen3/hpd/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/hpd/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2bbeb87205ee4b4f0af8e8976644241f3c08d44b
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:789675147525511f42d0da785a2d99158e9c329deecd6de057a2b56247966b58
+size 351841765
diff --git a/qwen3/hpd/checkpoint-1570/latest b/qwen3/hpd/checkpoint-1570/latest
new file mode 100644
index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/latest
@@ -0,0 +1 @@
+global_step1570
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-1570/rng_state_0.pth b/qwen3/hpd/checkpoint-1570/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3c3be262c1d807ed672ac8b4a568fd0a1afbe84c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aee6d60ff5f81fb764ba462b95ba7f011e50c79d398b05b7858a59089a924c78
+size 14917
diff --git a/qwen3/hpd/checkpoint-1570/rng_state_1.pth b/qwen3/hpd/checkpoint-1570/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f4f27481a367e3bc51ad2acad73eb23ed3288d01
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c77dd90b5daf4b2b93364e2e72a9c4e319ac7c5ee60e6ee049c01919a50d0fc2
+size 14917
diff --git a/qwen3/hpd/checkpoint-1570/scheduler.pt b/qwen3/hpd/checkpoint-1570/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae
+size 1465
diff --git a/qwen3/hpd/checkpoint-1570/tokenizer.json b/qwen3/hpd/checkpoint-1570/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/hpd/checkpoint-1570/tokenizer_config.json b/qwen3/hpd/checkpoint-1570/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/hpd/checkpoint-1570/trainer_state.json b/qwen3/hpd/checkpoint-1570/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..78428caa4589b624678f85a5c9a1d7e99b542a9d
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/trainer_state.json
@@ -0,0 +1,1900 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 1570,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "adv1_neg_pct": 0.2358350306749344,
+      "adv2_active_pct": 17.716575622558594,
+      "ce_loss": 0.6496895551681519,
+      "epoch": 0.06382130035899482,
+      "grad_norm": 4.9955339431762695,
+      "hpd_loss": 0.7524586915969849,
+      "k1_gt_raw_max": 0.3395599126815796,
+      "k1_gt_raw_mean": 0.08363302797079086,
+      "k1_gt_raw_min": -0.006161125842481852,
+      "k1_gt_raw_neg_pct": 0.2358350306749344,
+      "k1_sample_max": -1.2287571430206299,
+      "k1_sample_mean": -5.443553924560547,
+      "k1_sample_min": -17.516345977783203,
+      "k1_sample_neg_pct": 97.79017639160156,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.7524587631225585,
+      "mask3_pct": 17.716575622558594,
+      "neg_student_nll_loss": 0.6496895551681519,
+      "sampled_diff_pct": 17.718591690063477,
+      "step": 20
+    },
+    {
+      "adv1_neg_pct": 0.24569129943847656,
+      "adv2_active_pct": 6.35067081451416,
+      "ce_loss": 0.12904959917068481,
+      "epoch": 0.12764260071798963,
+      "grad_norm": 3.5179519653320312,
+      "hpd_loss": 0.07163389772176743,
+      "k1_gt_raw_max": 0.3387124538421631,
+      "k1_gt_raw_mean": 0.03725019469857216,
+      "k1_gt_raw_min": -0.009099474176764488,
+      "k1_gt_raw_neg_pct": 0.24569129943847656,
+      "k1_sample_max": -1.1939605474472046,
+      "k1_sample_mean": -4.561788082122803,
+      "k1_sample_min": -10.153608322143555,
+      "k1_sample_neg_pct": 96.875,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.07163387537002563,
+      "mask3_pct": 6.35067081451416,
+      "neg_student_nll_loss": 0.12904959917068481,
+      "sampled_diff_pct": 6.35067081451416,
+      "step": 40
+    },
+    {
+      "adv1_neg_pct": 0.18458101153373718,
+      "adv2_active_pct": 5.008382320404053,
+      "ce_loss": 0.10459579527378082,
+      "epoch": 0.19146390107698444,
+      "grad_norm": 2.890872001647949,
+      "hpd_loss": 0.09348365664482117,
+      "k1_gt_raw_max": 0.3109760880470276,
+      "k1_gt_raw_mean": 0.026082728058099747,
+      "k1_gt_raw_min": -0.01705370843410492,
+      "k1_gt_raw_neg_pct": 0.18458101153373718,
+      "k1_sample_max": -2.0660922527313232,
+      "k1_sample_mean": -4.749420166015625,
+      "k1_sample_min": -8.562150955200195,
+      "k1_sample_neg_pct": 89.6875,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.09348366260528565,
+      "mask3_pct": 5.008382320404053,
+      "neg_student_nll_loss": 0.10459579527378082,
+      "sampled_diff_pct": 5.008382320404053,
+      "step": 60
+    },
+    {
+      "adv1_neg_pct": 0.3009452819824219,
+      "adv2_active_pct": 3.9314260482788086,
+      "ce_loss": 0.08063317835330963,
+      "epoch": 0.25528520143597927,
+      "grad_norm": 2.033250570297241,
+      "hpd_loss": 0.08107596635818481,
+      "k1_gt_raw_max": 0.2881737947463989,
+      "k1_gt_raw_mean": 0.019415946677327156,
+      "k1_gt_raw_min": -0.008919830434024334,
+      "k1_gt_raw_neg_pct": 0.3009452819824219,
+      "k1_sample_max": -1.6159262657165527,
+      "k1_sample_mean": -3.7506155967712402,
+      "k1_sample_min": -6.360409259796143,
+      "k1_sample_neg_pct": 86.39583587646484,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.0810759723186493,
+      "mask3_pct": 3.9314260482788086,
+      "neg_student_nll_loss": 0.08063317835330963,
+      "sampled_diff_pct": 3.9378738403320312,
+      "step": 80
+    },
+    {
+      "adv1_neg_pct": 0.22911369800567627,
+      "adv2_active_pct": 2.694115161895752,
+      "ce_loss": 0.052447859197854996,
+      "epoch": 0.3191065017949741,
+      "grad_norm": 1.751448392868042,
+      "hpd_loss": 0.027988094836473465,
+      "k1_gt_raw_max": 0.3031776547431946,
+      "k1_gt_raw_mean": 0.01864592172205448,
+      "k1_gt_raw_min": -0.007533406838774681,
+      "k1_gt_raw_neg_pct": 0.22911369800567627,
+      "k1_sample_max": -1.57374906539917,
+      "k1_sample_mean": -3.1450629234313965,
+      "k1_sample_min": -5.145340919494629,
+      "k1_sample_neg_pct": 84.70052337646484,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.027988094091415405,
+      "mask3_pct": 2.6920177936553955,
+      "neg_student_nll_loss": 0.052447859197854996,
+      "sampled_diff_pct": 2.7128100395202637,
+      "step": 100
+    },
+    {
+      "adv1_neg_pct": 0.1952832043170929,
+      "adv2_active_pct": 3.306915044784546,
+      "ce_loss": 0.04980571195483208,
+      "epoch": 0.3829278021539689,
+      "grad_norm": 3.930035352706909,
+      "hpd_loss": 0.015151199884712696,
+      "k1_gt_raw_max": 0.28564560413360596,
+      "k1_gt_raw_mean": 0.01884515956044197,
+      "k1_gt_raw_min": -0.001684204675257206,
+      "k1_gt_raw_neg_pct": 0.1952832043170929,
+      "k1_sample_max": -1.9181127548217773,
+      "k1_sample_mean": -3.2553763389587402,
+      "k1_sample_min": -4.8449602127075195,
+      "k1_sample_neg_pct": 75.20833587646484,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.015151196718215942,
+      "mask3_pct": 3.306915044784546,
+      "neg_student_nll_loss": 0.04980571195483208,
+      "sampled_diff_pct": 3.3092129230499268,
+      "step": 120
+    },
+    {
+      "adv1_neg_pct": 0.3264433741569519,
+      "adv2_active_pct": 2.25433087348938,
+      "ce_loss": 0.04018855467438698,
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.6639359593391418,
+      "hpd_loss": 0.02619136869907379,
+      "k1_gt_raw_max": 0.24626323580741882,
+      "k1_gt_raw_mean": 0.015010751783847809,
+      "k1_gt_raw_min": -0.004701526835560799,
+      "k1_gt_raw_neg_pct": 0.3264433741569519,
+      "k1_sample_max": -1.902454137802124,
+      "k1_sample_mean": -2.805896282196045,
+      "k1_sample_min": -3.829012393951416,
+      "k1_sample_neg_pct": 71.25,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.026191365718841553,
+      "mask3_pct": 2.25433087348938,
+      "neg_student_nll_loss": 0.04018855467438698,
+      "sampled_diff_pct": 2.2565953731536865,
+      "step": 140
+    },
+    {
+      "adv1_neg_pct": 0.3137112259864807,
+      "adv2_active_pct": 1.3726557493209839,
+      "ce_loss": 0.03148110955953598,
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.548140525817871,
+      "hpd_loss": 0.03039727918803692,
+      "k1_gt_raw_max": 0.20344290137290955,
+      "k1_gt_raw_mean": 0.009032711386680603,
+      "k1_gt_raw_min": -0.0009857998229563236,
+      "k1_gt_raw_neg_pct": 0.3137112259864807,
+      "k1_sample_max": -2.153831958770752,
+      "k1_sample_mean": -2.81235933303833,
+      "k1_sample_min": -3.547999858856201,
+      "k1_sample_neg_pct": 58.75,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.03039727509021759,
+      "mask3_pct": 1.3726557493209839,
+      "neg_student_nll_loss": 0.03148110955953598,
+      "sampled_diff_pct": 1.3726557493209839,
+      "step": 160
+    },
+    {
+      "adv1_neg_pct": 0.4895250201225281,
+      "adv2_active_pct": 1.1766502857208252,
+      "ce_loss": 0.02162190154194832,
+      "epoch": 0.5743917032309533,
+      "grad_norm": 2.528654098510742,
+      "hpd_loss": 0.011955421417951584,
+      "k1_gt_raw_max": 0.20187309384346008,
+      "k1_gt_raw_mean": 0.008551979437470436,
+      "k1_gt_raw_min": -0.0070910281501710415,
+      "k1_gt_raw_neg_pct": 0.4895250201225281,
+      "k1_sample_max": -1.979987621307373,
+      "k1_sample_mean": -2.5711493492126465,
+      "k1_sample_min": -3.2561821937561035,
+      "k1_sample_neg_pct": 56.09375,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.011955419182777404,
+      "mask3_pct": 1.1766502857208252,
+      "neg_student_nll_loss": 0.02162190154194832,
+      "sampled_diff_pct": 1.185581922531128,
+      "step": 180
+    },
+    {
+      "adv1_neg_pct": 0.5023188591003418,
+      "adv2_active_pct": 1.5687706470489502,
+      "ce_loss": 0.027880553156137466,
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.9650227427482605,
+      "hpd_loss": 0.010340285487473011,
+      "k1_gt_raw_max": 0.2007327377796173,
+      "k1_gt_raw_mean": 0.009667925536632538,
+      "k1_gt_raw_min": -0.01589183136820793,
+      "k1_gt_raw_neg_pct": 0.5023188591003418,
+      "k1_sample_max": -1.5354162454605103,
+      "k1_sample_mean": -2.1268630027770996,
+      "k1_sample_min": -2.771559238433838,
+      "k1_sample_neg_pct": 56.40625,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.010340283811092376,
+      "mask3_pct": 1.5687706470489502,
+      "neg_student_nll_loss": 0.027880553156137466,
+      "sampled_diff_pct": 1.5725816488265991,
+      "step": 200
+    },
+    {
+      "adv1_neg_pct": 0.48300591111183167,
+      "adv2_active_pct": 1.1448569297790527,
+      "ce_loss": 0.023295864462852478,
+      "epoch": 0.702034303948943,
+      "grad_norm": 2.59177303314209,
+      "hpd_loss": 0.019972147420048714,
+      "k1_gt_raw_max": 0.18090441823005676,
+      "k1_gt_raw_mean": 0.0075406162068247795,
+      "k1_gt_raw_min": -0.014693280681967735,
+      "k1_gt_raw_neg_pct": 0.48300591111183167,
+      "k1_sample_max": -1.6416490077972412,
+      "k1_sample_mean": -2.2296323776245117,
+      "k1_sample_min": -2.930915355682373,
+      "k1_sample_neg_pct": 48.515625,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01997213959693909,
+      "mask3_pct": 1.1448569297790527,
+      "neg_student_nll_loss": 0.023295864462852478,
+      "sampled_diff_pct": 1.154510736465454,
+      "step": 220
+    },
+    {
+      "adv1_neg_pct": 0.765082597732544,
+      "adv2_active_pct": 0.9244414567947388,
+      "ce_loss": 0.02368738129734993,
+      "epoch": 0.7658556043079378,
+      "grad_norm": 1.1686080694198608,
+      "hpd_loss": 0.02641882374882698,
+      "k1_gt_raw_max": 0.1654866635799408,
+      "k1_gt_raw_mean": 0.005874675698578358,
+      "k1_gt_raw_min": -0.009244760498404503,
+      "k1_gt_raw_neg_pct": 0.765082597732544,
+      "k1_sample_max": -1.3134702444076538,
+      "k1_sample_mean": -1.9339661598205566,
+      "k1_sample_min": -2.6181559562683105,
+      "k1_sample_neg_pct": 45.3125,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.026418817043304444,
+      "mask3_pct": 0.9244414567947388,
+      "neg_student_nll_loss": 0.02368738129734993,
+      "sampled_diff_pct": 0.9244414567947388,
+      "step": 240
+    },
+    {
+      "adv1_neg_pct": 0.8438892364501953,
+      "adv2_active_pct": 0.929736852645874,
+      "ce_loss": 0.020094729959964752,
+      "epoch": 0.8296769046669326,
+      "grad_norm": 1.2779905796051025,
+      "hpd_loss": 0.015618223696947098,
+      "k1_gt_raw_max": 0.16853322088718414,
+      "k1_gt_raw_mean": 0.006190822925418615,
+      "k1_gt_raw_min": -0.009831264615058899,
+      "k1_gt_raw_neg_pct": 0.8438892364501953,
+      "k1_sample_max": -1.3172353506088257,
+      "k1_sample_mean": -1.7796552181243896,
+      "k1_sample_min": -2.261643409729004,
+      "k1_sample_neg_pct": 44.95738983154297,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.015618222951889037,
+      "mask3_pct": 0.9239488840103149,
+      "neg_student_nll_loss": 0.020094729959964752,
+      "sampled_diff_pct": 0.9378187656402588,
+      "step": 260
+    },
+    {
+      "adv1_neg_pct": 0.9945487976074219,
+      "adv2_active_pct": 0.7392160892486572,
+      "ce_loss": 0.015887977555394173,
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.26237931847572327,
+      "hpd_loss": 0.015452454797923565,
+      "k1_gt_raw_max": 0.14525339007377625,
+      "k1_gt_raw_mean": 0.0046372562646865845,
+      "k1_gt_raw_min": -0.003686224576085806,
+      "k1_gt_raw_neg_pct": 0.9945487976074219,
+      "k1_sample_max": -1.206074595451355,
+      "k1_sample_mean": -1.4332499504089355,
+      "k1_sample_min": -1.671927571296692,
+      "k1_sample_neg_pct": 35.625,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.015452450513839722,
+      "mask3_pct": 0.7392160892486572,
+      "neg_student_nll_loss": 0.015887977555394173,
+      "sampled_diff_pct": 0.7392160892486572,
+      "step": 280
+    },
+    {
+      "adv1_neg_pct": 0.9700182676315308,
+      "adv2_active_pct": 0.6018879413604736,
+      "ce_loss": 0.014513559639453888,
+      "epoch": 0.9573195053849222,
+      "grad_norm": 2.0338079929351807,
+      "hpd_loss": 0.013913260772824287,
+      "k1_gt_raw_max": 0.13846856355667114,
+      "k1_gt_raw_mean": 0.004541991278529167,
+      "k1_gt_raw_min": -0.002533096820116043,
+      "k1_gt_raw_neg_pct": 0.9700182676315308,
+      "k1_sample_max": -1.2417628765106201,
+      "k1_sample_mean": -1.4318351745605469,
+      "k1_sample_min": -1.6163195371627808,
+      "k1_sample_neg_pct": 32.864585876464844,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013913258910179138,
+      "mask3_pct": 0.6018879413604736,
+      "neg_student_nll_loss": 0.014513559639453888,
+      "sampled_diff_pct": 0.607985258102417,
+      "step": 300
+    },
+    {
+      "adv1_neg_pct": 0.8508870601654053,
+      "adv2_active_pct": 0.859713077545166,
+      "ce_loss": 0.019685909152030945,
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 86.6822,
+      "eval_generator_rouge1": 99.4006,
+      "eval_generator_rouge2": 99.0209,
+      "eval_generator_rougeL": 98.951,
+      "eval_loss": 0.010082065127789974,
+      "eval_runtime": 595.0313,
+      "eval_samples_per_second": 5.738,
+      "eval_selector_accuracy": 93.9836,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.18,
+      "hpd_loss": 0.02267773076891899,
+      "k1_gt_raw_max": 0.14172008633613586,
+      "k1_gt_raw_mean": 0.0054550133645534515,
+      "k1_gt_raw_min": -0.005414052866399288,
+      "k1_gt_raw_neg_pct": 0.8508870601654053,
+      "k1_sample_max": -1.4249026775360107,
+      "k1_sample_mean": -1.6359399557113647,
+      "k1_sample_min": -1.8393499851226807,
+      "k1_sample_neg_pct": 36.4485969543457,
+      "mask3_pct": 0.859713077545166,
+      "neg_student_nll_loss": 0.019685909152030945,
+      "sampled_diff_pct": 0.859713077545166,
+      "step": 314
+    },
+    {
+      "adv1_neg_pct": 0.6673219203948975,
+      "adv2_active_pct": 1.2173185348510742,
+      "ce_loss": 0.011882005259394646,
+      "epoch": 1.0191463901076985,
+      "grad_norm": 4.386560916900635,
+      "hpd_loss": -0.0019697409588843584,
+      "k1_gt_raw_max": 0.14985336363315582,
+      "k1_gt_raw_mean": 0.006570600904524326,
+      "k1_gt_raw_min": -0.0011412458261474967,
+      "k1_gt_raw_neg_pct": 0.6673219203948975,
+      "k1_sample_max": -0.5733932256698608,
+      "k1_sample_mean": -0.7650949358940125,
+      "k1_sample_min": -0.9706472158432007,
+      "k1_sample_neg_pct": 36.458335876464844,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014574740827083588,
+      "mask3_pct": 1.2173185348510742,
+      "neg_student_nll_loss": 0.011882005259394646,
+      "sampled_diff_pct": 1.2173185348510742,
+      "step": 320
+    },
+    {
+      "adv1_neg_pct": 1.1466679573059082,
+      "adv2_active_pct": 0.4476942718029022,
+      "ce_loss": 0.009723825380206108,
+      "epoch": 1.0829676904666932,
+      "grad_norm": 1.2898244857788086,
+      "hpd_loss": 0.009684150107204914,
+      "k1_gt_raw_max": 0.10980867594480515,
+      "k1_gt_raw_mean": 0.0032046493142843246,
+      "k1_gt_raw_min": -0.009925473481416702,
+      "k1_gt_raw_neg_pct": 1.1466679573059082,
+      "k1_sample_max": -0.76816326379776,
+      "k1_sample_mean": -0.8626709580421448,
+      "k1_sample_min": -0.9573334455490112,
+      "k1_sample_neg_pct": 28.95833396911621,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.0096841499209404,
+      "mask3_pct": 0.4476942718029022,
+      "neg_student_nll_loss": 0.009723825380206108,
+      "sampled_diff_pct": 0.4520599842071533,
+      "step": 340
+    },
+    {
+      "adv1_neg_pct": 1.657383918762207,
+      "adv2_active_pct": 0.7416136860847473,
+      "ce_loss": 0.01363883912563324,
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.8189982175827026,
+      "hpd_loss": 0.012504098936915398,
+      "k1_gt_raw_max": 0.11255955696105957,
+      "k1_gt_raw_mean": 0.0030380175448954105,
+      "k1_gt_raw_min": -0.01135759986937046,
+      "k1_gt_raw_neg_pct": 1.657383918762207,
+      "k1_sample_max": -1.1351618766784668,
+      "k1_sample_mean": -1.3799619674682617,
+      "k1_sample_min": -1.6486986875534058,
+      "k1_sample_neg_pct": 30.0,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.012504099309444428,
+      "mask3_pct": 0.7416136860847473,
+      "neg_student_nll_loss": 0.01363883912563324,
+      "sampled_diff_pct": 0.7438143491744995,
+      "step": 360
+    },
+    {
+      "adv1_neg_pct": 1.3510011434555054,
+      "adv2_active_pct": 0.3445201814174652,
+      "ce_loss": 0.007902171462774277,
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.5892553329467773,
+      "hpd_loss": 0.008744428865611553,
+      "k1_gt_raw_max": 0.09419713914394379,
+      "k1_gt_raw_mean": 0.0022562379017472267,
+      "k1_gt_raw_min": -0.006403174251317978,
+      "k1_gt_raw_neg_pct": 1.3510011434555054,
+      "k1_sample_max": -0.583869457244873,
+      "k1_sample_mean": -0.7341683506965637,
+      "k1_sample_min": -0.8994753360748291,
+      "k1_sample_neg_pct": 23.75,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.008744429051876067,
+      "mask3_pct": 0.3445201814174652,
+      "neg_student_nll_loss": 0.007902171462774277,
+      "sampled_diff_pct": 0.35682976245880127,
+      "step": 380
+    },
+    {
+      "adv1_neg_pct": 1.4141113758087158,
+      "adv2_active_pct": 0.3374101221561432,
+      "ce_loss": 0.008193380199372768,
+      "epoch": 1.2744315915436777,
+      "grad_norm": 1.2929012775421143,
+      "hpd_loss": 0.006689602509140968,
+      "k1_gt_raw_max": 0.10671139508485794,
+      "k1_gt_raw_mean": 0.002954404801130295,
+      "k1_gt_raw_min": -0.006222248077392578,
+      "k1_gt_raw_neg_pct": 1.4141114950180054,
+      "k1_sample_max": -0.6661227941513062,
+      "k1_sample_mean": -0.7585095763206482,
+      "k1_sample_min": -0.8607568740844727,
+      "k1_sample_neg_pct": 25.760417938232422,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.0066896028816699985,
+      "mask3_pct": 0.3344895839691162,
+      "neg_student_nll_loss": 0.008193380199372768,
+      "sampled_diff_pct": 0.3449546694755554,
+      "step": 400
+    },
+    {
+      "adv1_neg_pct": 2.1220993995666504,
+      "adv2_active_pct": 0.4313049912452698,
+      "ce_loss": 0.011112527921795845,
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.8164143562316895,
+      "hpd_loss": 0.010808300226926804,
+      "k1_gt_raw_max": 0.10569541901350021,
+      "k1_gt_raw_mean": 0.0035689054057002068,
+      "k1_gt_raw_min": -0.004470471758395433,
+      "k1_gt_raw_neg_pct": 2.1220993995666504,
+      "k1_sample_max": -0.6986867189407349,
+      "k1_sample_mean": -0.7543543577194214,
+      "k1_sample_min": -0.8270018100738525,
+      "k1_sample_neg_pct": 25.279605865478516,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.010808297991752624,
+      "mask3_pct": 0.4313049912452698,
+      "neg_student_nll_loss": 0.011112527921795845,
+      "sampled_diff_pct": 0.4342671036720276,
+      "step": 420
+    },
+    {
+      "adv1_neg_pct": 1.9999275207519531,
+      "adv2_active_pct": 0.2967550456523895,
+      "ce_loss": 0.007173327729105949,
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.959374189376831,
+      "hpd_loss": 0.005844146944582462,
+      "k1_gt_raw_max": 0.08694229274988174,
+      "k1_gt_raw_mean": 0.002139701507985592,
+      "k1_gt_raw_min": -0.005928537808358669,
+      "k1_gt_raw_neg_pct": 1.9999275207519531,
+      "k1_sample_max": -0.8034830093383789,
+      "k1_sample_mean": -0.8980427980422974,
+      "k1_sample_min": -0.9926026463508606,
+      "k1_sample_neg_pct": 20.78125,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.005844147875905037,
+      "mask3_pct": 0.2967550456523895,
+      "neg_student_nll_loss": 0.007173327729105949,
+      "sampled_diff_pct": 0.30670204758644104,
+      "step": 440
+    },
+    {
+      "adv1_neg_pct": 1.9445667266845703,
+      "adv2_active_pct": 0.28356558084487915,
+      "ce_loss": 0.007787229493260384,
+      "epoch": 1.4658954926206622,
+      "grad_norm": 1.3586090803146362,
+      "hpd_loss": 0.009073459543287754,
+      "k1_gt_raw_max": 0.07425311952829361,
+      "k1_gt_raw_mean": 0.0016224482096731663,
+      "k1_gt_raw_min": -0.0022226697765290737,
+      "k1_gt_raw_neg_pct": 1.9445667266845703,
+      "k1_sample_max": -0.6454492807388306,
+      "k1_sample_mean": -0.686149001121521,
+      "k1_sample_min": -0.7300035953521729,
+      "k1_sample_neg_pct": 17.1875,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.009073460102081298,
+      "mask3_pct": 0.27911925315856934,
+      "neg_student_nll_loss": 0.007787229493260384,
+      "sampled_diff_pct": 0.28356558084487915,
+      "step": 460
+    },
+    {
+      "adv1_neg_pct": 1.8402546644210815,
+      "adv2_active_pct": 0.41872814297676086,
+      "ce_loss": 0.005175907164812088,
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.2023886740207672,
+      "hpd_loss": 0.0015706174308434129,
+      "k1_gt_raw_max": 0.07684825360774994,
+      "k1_gt_raw_mean": 0.0022208630107343197,
+      "k1_gt_raw_min": -0.008054896257817745,
+      "k1_gt_raw_neg_pct": 1.8402546644210815,
+      "k1_sample_max": -0.47196078300476074,
+      "k1_sample_mean": -0.4973880648612976,
+      "k1_sample_min": -0.5228154063224792,
+      "k1_sample_neg_pct": 17.5,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0015706172212958335,
+      "mask3_pct": 0.41872814297676086,
+      "neg_student_nll_loss": 0.005175907164812088,
+      "sampled_diff_pct": 0.4208254814147949,
+      "step": 480
+    },
+    {
+      "adv1_neg_pct": 2.5322799682617188,
+      "adv2_active_pct": 0.33805567026138306,
+      "ce_loss": 0.00585869699716568,
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.030877001583576202,
+      "hpd_loss": 0.003286010818555951,
+      "k1_gt_raw_max": 0.058261577039957047,
+      "k1_gt_raw_mean": 0.002064550295472145,
+      "k1_gt_raw_min": -0.0029995564837008715,
+      "k1_gt_raw_neg_pct": 2.5322799682617188,
+      "k1_sample_max": -0.3366956412792206,
+      "k1_sample_mean": -0.39731863141059875,
+      "k1_sample_min": -0.45794159173965454,
+      "k1_sample_neg_pct": 14.0625,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.0032860103994607924,
+      "mask3_pct": 0.33805567026138306,
+      "neg_student_nll_loss": 0.00585869699716568,
+      "sampled_diff_pct": 0.3419136703014374,
+      "step": 500
+    },
+    {
+      "adv1_neg_pct": 2.701911449432373,
+      "adv2_active_pct": 0.3643760681152344,
+      "ce_loss": 0.010007714852690697,
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.14292795956134796,
+      "hpd_loss": 0.012855440378189087,
+      "k1_gt_raw_max": 0.06845426559448242,
+      "k1_gt_raw_mean": 0.0017418819479644299,
+      "k1_gt_raw_min": -0.004030767362564802,
+      "k1_gt_raw_neg_pct": 2.701911449432373,
+      "k1_sample_max": -0.4802917242050171,
+      "k1_sample_mean": -0.5421380996704102,
+      "k1_sample_min": -0.6135058999061584,
+      "k1_sample_neg_pct": 17.03125,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.012855446338653565,
+      "mask3_pct": 0.3643760681152344,
+      "neg_student_nll_loss": 0.010007714852690697,
+      "sampled_diff_pct": 0.3710411787033081,
+      "step": 520
+    },
+    {
+      "adv1_neg_pct": 2.248600959777832,
+      "adv2_active_pct": 0.5108850598335266,
+      "ce_loss": 0.01003289595246315,
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.6681362390518188,
+      "hpd_loss": 0.009245548397302628,
+      "k1_gt_raw_max": 0.08858497440814972,
+      "k1_gt_raw_mean": 0.002853624988347292,
+      "k1_gt_raw_min": -0.004460392985492945,
+      "k1_gt_raw_neg_pct": 2.248600959777832,
+      "k1_sample_max": -0.7408795356750488,
+      "k1_sample_mean": -0.8169400691986084,
+      "k1_sample_min": -0.8906024694442749,
+      "k1_sample_neg_pct": 22.65625,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.009245550632476807,
+      "mask3_pct": 0.5108850598335266,
+      "neg_student_nll_loss": 0.01003289595246315,
+      "sampled_diff_pct": 0.5210334062576294,
+      "step": 540
+    },
+    {
+      "adv1_neg_pct": 2.4635496139526367,
+      "adv2_active_pct": 0.22115065157413483,
+      "ce_loss": 0.005004607141017914,
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.08243642747402191,
+      "hpd_loss": 0.0062531642615795135,
+      "k1_gt_raw_max": 0.0580272376537323,
+      "k1_gt_raw_mean": 0.0011763360816985369,
+      "k1_gt_raw_min": -0.005644598975777626,
+      "k1_gt_raw_neg_pct": 2.4635493755340576,
+      "k1_sample_max": -0.2404015064239502,
+      "k1_sample_mean": -0.2807300090789795,
+      "k1_sample_min": -0.32241004705429077,
+      "k1_sample_neg_pct": 14.921875,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.006253162026405334,
+      "mask3_pct": 0.22115065157413483,
+      "neg_student_nll_loss": 0.005004607141017914,
+      "sampled_diff_pct": 0.22733174264431,
+      "step": 560
+    },
+    {
+      "adv1_neg_pct": 3.3381173610687256,
+      "adv2_active_pct": 0.23258033394813538,
+      "ce_loss": 0.006839662324637175,
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.0869053527712822,
+      "hpd_loss": 0.00969111267477274,
+      "k1_gt_raw_max": 0.05798381567001343,
+      "k1_gt_raw_mean": 0.0016341875307261944,
+      "k1_gt_raw_min": -0.009443331509828568,
+      "k1_gt_raw_neg_pct": 3.3381173610687256,
+      "k1_sample_max": -0.29867684841156006,
+      "k1_sample_mean": -0.31859445571899414,
+      "k1_sample_min": -0.338512122631073,
+      "k1_sample_neg_pct": 10.625,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.009691111743450165,
+      "mask3_pct": 0.23258033394813538,
+      "neg_student_nll_loss": 0.006839662324637175,
+      "sampled_diff_pct": 0.24274389445781708,
+      "step": 580
+    },
+    {
+      "adv1_neg_pct": 4.040760040283203,
+      "adv2_active_pct": 0.20796719193458557,
+      "ce_loss": 0.002632163930684328,
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2512037456035614,
+      "hpd_loss": 0.0005975529784336686,
+      "k1_gt_raw_max": 0.049394719302654266,
+      "k1_gt_raw_mean": 0.0014062562258914113,
+      "k1_gt_raw_min": -0.002836518455296755,
+      "k1_gt_raw_neg_pct": 4.040760040283203,
+      "k1_sample_max": -0.23663434386253357,
+      "k1_sample_mean": -0.255886435508728,
+      "k1_sample_min": -0.2802615463733673,
+      "k1_sample_neg_pct": 10.208333969116211,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.0005975536536425352,
+      "mask3_pct": 0.20796719193458557,
+      "neg_student_nll_loss": 0.002632163930684328,
+      "sampled_diff_pct": 0.21867018938064575,
+      "step": 600
+    },
+    {
+      "adv1_neg_pct": 4.097904205322266,
+      "adv2_active_pct": 0.1848699152469635,
+      "ce_loss": 0.0032025063410401344,
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.338096022605896,
+      "hpd_loss": 0.002149906475096941,
+      "k1_gt_raw_max": 0.043635837733745575,
+      "k1_gt_raw_mean": 0.0007720005232840776,
+      "k1_gt_raw_min": -0.0036990540102124214,
+      "k1_gt_raw_neg_pct": 4.097904205322266,
+      "k1_sample_max": -0.2395995557308197,
+      "k1_sample_mean": -0.2737259864807129,
+      "k1_sample_min": -0.3078524172306061,
+      "k1_sample_neg_pct": 13.4375,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.0021499061957001688,
+      "mask3_pct": 0.1848699152469635,
+      "neg_student_nll_loss": 0.0032025063410401344,
+      "sampled_diff_pct": 0.18927133083343506,
+      "step": 620
+    },
+    {
+      "adv1_neg_pct": 3.390739917755127,
+      "adv2_active_pct": 0.10791555047035217,
+      "ce_loss": 0.001353576430119574,
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.1519,
+      "eval_generator_rouge1": 99.7488,
+      "eval_generator_rouge2": 99.6229,
+      "eval_generator_rougeL": 99.5782,
+      "eval_loss": 0.004672760609537363,
+      "eval_runtime": 596.1717,
+      "eval_samples_per_second": 5.727,
+      "eval_selector_accuracy": 98.014,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.179,
+      "hpd_loss": 0.0011793917510658503,
+      "k1_gt_raw_max": 0.03581507131457329,
+      "k1_gt_raw_mean": 0.0006900117732584476,
+      "k1_gt_raw_min": -0.002846459159627557,
+      "k1_gt_raw_neg_pct": 3.390739917755127,
+      "k1_sample_max": -0.06917751580476761,
+      "k1_sample_mean": -0.06961424648761749,
+      "k1_sample_min": -0.07005097717046738,
+      "k1_sample_neg_pct": 10.1694917678833,
+      "mask3_pct": 0.10791555047035217,
+      "neg_student_nll_loss": 0.001353576430119574,
+      "sampled_diff_pct": 0.10791555047035217,
+      "step": 628
+    },
+    {
+      "adv1_neg_pct": 3.476264715194702,
+      "adv2_active_pct": 0.12990054488182068,
+      "ce_loss": 0.004336793441325426,
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.6891085505485535,
+      "hpd_loss": 0.002609923481941223,
+      "k1_gt_raw_max": 0.038067810237407684,
+      "k1_gt_raw_mean": 0.0007896563620306551,
+      "k1_gt_raw_min": -0.005201837047934532,
+      "k1_gt_raw_neg_pct": 3.476264715194702,
+      "k1_sample_max": -0.2407418042421341,
+      "k1_sample_mean": -0.2449159324169159,
+      "k1_sample_min": -0.2522057890892029,
+      "k1_sample_neg_pct": 9.798177719116211,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.0020211638882756235,
+      "mask3_pct": 0.12990054488182068,
+      "neg_student_nll_loss": 0.004336793441325426,
+      "sampled_diff_pct": 0.14956170320510864,
+      "step": 640
+    },
+    {
+      "adv1_neg_pct": 3.8216357231140137,
+      "adv2_active_pct": 0.16393554210662842,
+      "ce_loss": 0.0034592957235872746,
+      "epoch": 2.102114080574392,
+      "grad_norm": 1.311052918434143,
+      "hpd_loss": 0.004151628818362951,
+      "k1_gt_raw_max": 0.04581638425588608,
+      "k1_gt_raw_mean": 0.0008853404433466494,
+      "k1_gt_raw_min": -0.0063157654367387295,
+      "k1_gt_raw_neg_pct": 3.8216357231140137,
+      "k1_sample_max": -0.250868022441864,
+      "k1_sample_mean": -0.30226844549179077,
+      "k1_sample_min": -0.3555181622505188,
+      "k1_sample_neg_pct": 10.833333969116211,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.004151628911495208,
+      "mask3_pct": 0.16393554210662842,
+      "neg_student_nll_loss": 0.0034592957235872746,
+      "sampled_diff_pct": 0.16612085700035095,
+      "step": 660
+    },
+    {
+      "adv1_neg_pct": 4.672894477844238,
+      "adv2_active_pct": 0.17785239219665527,
+      "ce_loss": 0.0037891073152422905,
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.20577986538410187,
+      "hpd_loss": 0.004049420822411776,
+      "k1_gt_raw_max": 0.047759026288986206,
+      "k1_gt_raw_mean": 0.0010929291602224112,
+      "k1_gt_raw_min": -0.008511744439601898,
+      "k1_gt_raw_neg_pct": 4.672894477844238,
+      "k1_sample_max": -0.2957162857055664,
+      "k1_sample_mean": -0.3683391511440277,
+      "k1_sample_min": -0.440962016582489,
+      "k1_sample_neg_pct": 12.8125,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.004049421846866607,
+      "mask3_pct": 0.17785239219665527,
+      "neg_student_nll_loss": 0.0037891073152422905,
+      "sampled_diff_pct": 0.17980551719665527,
+      "step": 680
+    },
+    {
+      "adv1_neg_pct": 3.504452705383301,
+      "adv2_active_pct": 0.33364731073379517,
+      "ce_loss": 0.00837249867618084,
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.05891747772693634,
+      "hpd_loss": 0.012418361380696297,
+      "k1_gt_raw_max": 0.044552117586135864,
+      "k1_gt_raw_mean": 0.001460403436794877,
+      "k1_gt_raw_min": -0.0013761408627033234,
+      "k1_gt_raw_neg_pct": 3.504452705383301,
+      "k1_sample_max": -0.41403040289878845,
+      "k1_sample_mean": -0.4155550003051758,
+      "k1_sample_min": -0.4170795679092407,
+      "k1_sample_neg_pct": 12.8125,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.012418360263109208,
+      "mask3_pct": 0.33364731073379517,
+      "neg_student_nll_loss": 0.00837249867618084,
+      "sampled_diff_pct": 0.33364731073379517,
+      "step": 700
+    },
+    {
+      "adv1_neg_pct": 2.9557998180389404,
+      "adv2_active_pct": 0.2356344759464264,
+      "ce_loss": 0.002732612658292055,
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.03069828823208809,
+      "hpd_loss": 0.0013308569323271513,
+      "k1_gt_raw_max": 0.043037086725234985,
+      "k1_gt_raw_mean": 0.001189029891975224,
+      "k1_gt_raw_min": -0.00335265276953578,
+      "k1_gt_raw_neg_pct": 2.9557998180389404,
+      "k1_sample_max": -0.12545080482959747,
+      "k1_sample_mean": -0.13167737424373627,
+      "k1_sample_min": -0.13885264098644257,
+      "k1_sample_neg_pct": 12.1875,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.0013308576308190823,
+      "mask3_pct": 0.2356344759464264,
+      "neg_student_nll_loss": 0.002732612658292055,
+      "sampled_diff_pct": 0.24056380987167358,
+      "step": 720
+    },
+    {
+      "adv1_neg_pct": 4.362781524658203,
+      "adv2_active_pct": 0.14987114071846008,
+      "ce_loss": 0.00366773409768939,
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.03650576248764992,
+      "hpd_loss": 0.0028897470328956842,
+      "k1_gt_raw_max": 0.03678428381681442,
+      "k1_gt_raw_mean": 0.000713786983396858,
+      "k1_gt_raw_min": -0.007670572958886623,
+      "k1_gt_raw_neg_pct": 4.362782001495361,
+      "k1_sample_max": -0.16289430856704712,
+      "k1_sample_mean": -0.19424015283584595,
+      "k1_sample_min": -0.2255859375,
+      "k1_sample_neg_pct": 9.0625,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.002889746427536011,
+      "mask3_pct": 0.14987114071846008,
+      "neg_student_nll_loss": 0.00366773409768939,
+      "sampled_diff_pct": 0.1667182892560959,
+      "step": 740
+    },
+    {
+      "adv1_neg_pct": 4.3088059425354,
+      "adv2_active_pct": 0.073929063975811,
+      "ce_loss": 0.0018987958319485188,
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.03706243261694908,
+      "hpd_loss": 0.001820039818994701,
+      "k1_gt_raw_max": 0.030663518235087395,
+      "k1_gt_raw_mean": 0.0005116415559314191,
+      "k1_gt_raw_min": -0.003718154039233923,
+      "k1_gt_raw_neg_pct": 4.3088059425354,
+      "k1_sample_max": -0.11870913207530975,
+      "k1_sample_mean": -0.12011738121509552,
+      "k1_sample_min": -0.12152564525604248,
+      "k1_sample_neg_pct": 7.03125,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.0018200401216745376,
+      "mask3_pct": 0.073929063975811,
+      "neg_student_nll_loss": 0.0018987958319485188,
+      "sampled_diff_pct": 0.08056812733411789,
+      "step": 760
+    },
+    {
+      "adv1_neg_pct": 4.6282758712768555,
+      "adv2_active_pct": 0.08583934605121613,
+      "ce_loss": 0.000978614203631878,
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.0633927583694458,
+      "hpd_loss": 0.0008434242918156087,
+      "k1_gt_raw_max": 0.02713455632328987,
+      "k1_gt_raw_mean": 0.0004272414662409574,
+      "k1_gt_raw_min": -0.006445502862334251,
+      "k1_gt_raw_neg_pct": 4.628275394439697,
+      "k1_sample_max": -0.10302496701478958,
+      "k1_sample_mean": -0.11016625910997391,
+      "k1_sample_min": -0.11730755120515823,
+      "k1_sample_neg_pct": 8.90625,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0008434242568910122,
+      "mask3_pct": 0.08583934605121613,
+      "neg_student_nll_loss": 0.000978614203631878,
+      "sampled_diff_pct": 0.09829449653625488,
+      "step": 780
+    },
+    {
+      "adv1_neg_pct": 5.204377174377441,
+      "adv2_active_pct": 0.15303727984428406,
+      "ce_loss": 0.004278196021914482,
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.2113279402256012,
+      "hpd_loss": 0.00520919868722558,
+      "k1_gt_raw_max": 0.0305880606174469,
+      "k1_gt_raw_mean": 0.0005127020995132625,
+      "k1_gt_raw_min": -0.0020859134383499622,
+      "k1_gt_raw_neg_pct": 5.204377174377441,
+      "k1_sample_max": -0.07243171334266663,
+      "k1_sample_mean": -0.08607004582881927,
+      "k1_sample_min": -0.10009756684303284,
+      "k1_sample_neg_pct": 10.0,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.005209198221564293,
+      "mask3_pct": 0.15303727984428406,
+      "neg_student_nll_loss": 0.004278196021914482,
+      "sampled_diff_pct": 0.1585448682308197,
+      "step": 800
+    },
+    {
+      "adv1_neg_pct": 5.268265724182129,
+      "adv2_active_pct": 0.10696102678775787,
+      "ce_loss": 0.003248082473874092,
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.32251229882240295,
+      "hpd_loss": 0.002392602153122425,
+      "k1_gt_raw_max": 0.03637195751070976,
+      "k1_gt_raw_mean": 0.0010833492269739509,
+      "k1_gt_raw_min": -0.0027411947958171368,
+      "k1_gt_raw_neg_pct": 5.268265724182129,
+      "k1_sample_max": -0.06464927643537521,
+      "k1_sample_mean": -0.06609892845153809,
+      "k1_sample_min": -0.06948131322860718,
+      "k1_sample_neg_pct": 6.826923370361328,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.0023926014080643656,
+      "mask3_pct": 0.1040404736995697,
+      "neg_student_nll_loss": 0.003248082473874092,
+      "sampled_diff_pct": 0.11661312729120255,
+      "step": 820
+    },
+    {
+      "adv1_neg_pct": 5.766000270843506,
+      "adv2_active_pct": 0.06631548702716827,
+      "ce_loss": 0.0020227693021297455,
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.2545914947986603,
+      "hpd_loss": 0.002442866563796997,
+      "k1_gt_raw_max": 0.027828488498926163,
+      "k1_gt_raw_mean": 0.0007106282282620668,
+      "k1_gt_raw_min": -0.004438548348844051,
+      "k1_gt_raw_neg_pct": 5.766000270843506,
+      "k1_sample_max": -0.10070931911468506,
+      "k1_sample_mean": -0.10070931911468506,
+      "k1_sample_min": -0.10070931911468506,
+      "k1_sample_neg_pct": 7.1875,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.002442866563796997,
+      "mask3_pct": 0.06631548702716827,
+      "neg_student_nll_loss": 0.0020227693021297455,
+      "sampled_diff_pct": 0.07493838667869568,
+      "step": 840
+    },
+    {
+      "adv1_neg_pct": 6.06646728515625,
+      "adv2_active_pct": 0.10010633617639542,
+      "ce_loss": 0.0019979095086455345,
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.2798786759376526,
+      "hpd_loss": 0.0019781002774834633,
+      "k1_gt_raw_max": 0.0327674075961113,
+      "k1_gt_raw_mean": 0.0005696950247511268,
+      "k1_gt_raw_min": -0.003440997563302517,
+      "k1_gt_raw_neg_pct": 6.06646728515625,
+      "k1_sample_max": -0.08324240148067474,
+      "k1_sample_mean": -0.13795003294944763,
+      "k1_sample_min": -0.1926576793193817,
+      "k1_sample_neg_pct": 9.375,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.0019781002774834633,
+      "mask3_pct": 0.10010633617639542,
+      "neg_student_nll_loss": 0.0019979095086455345,
+      "sampled_diff_pct": 0.10264699161052704,
+      "step": 860
+    },
+    {
+      "adv1_neg_pct": 5.891697883605957,
+      "adv2_active_pct": 0.07234375178813934,
+      "ce_loss": 0.0014330728445202112,
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.09429220855236053,
+      "hpd_loss": 0.0010787295177578926,
+      "k1_gt_raw_max": 0.038590773940086365,
+      "k1_gt_raw_mean": 0.0007731987861916423,
+      "k1_gt_raw_min": -0.0038950724992901087,
+      "k1_gt_raw_neg_pct": 5.891697883605957,
+      "k1_sample_max": -0.1224956065416336,
+      "k1_sample_mean": -0.1250840276479721,
+      "k1_sample_min": -0.12767243385314941,
+      "k1_sample_neg_pct": 7.34375,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.001078729424625635,
+      "mask3_pct": 0.07234375178813934,
+      "neg_student_nll_loss": 0.0014330728445202112,
+      "sampled_diff_pct": 0.07657574117183685,
+      "step": 880
+    },
+    {
+      "adv1_neg_pct": 6.011615753173828,
+      "adv2_active_pct": 0.09838910400867462,
+      "ce_loss": 0.0020957919768989086,
+      "epoch": 2.8679696848823295,
+      "grad_norm": 1.0214177370071411,
+      "hpd_loss": 0.0015987120568752289,
+      "k1_gt_raw_max": 0.03688127547502518,
+      "k1_gt_raw_mean": 0.0006160076009109616,
+      "k1_gt_raw_min": -0.005018594674766064,
+      "k1_gt_raw_neg_pct": 6.011615753173828,
+      "k1_sample_max": -0.2794632613658905,
+      "k1_sample_mean": -0.2854064106941223,
+      "k1_sample_min": -0.2913495600223541,
+      "k1_sample_neg_pct": 8.75,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.0015987120568752289,
+      "mask3_pct": 0.09838910400867462,
+      "neg_student_nll_loss": 0.0020957919768989086,
+      "sampled_diff_pct": 0.10469574481248856,
+      "step": 900
+    },
+    {
+      "adv1_neg_pct": 5.8451714515686035,
+      "adv2_active_pct": 0.12536077201366425,
+      "ce_loss": 0.0031956052407622337,
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.009789629839360714,
+      "hpd_loss": 0.005076252389699221,
+      "k1_gt_raw_max": 0.03244764730334282,
+      "k1_gt_raw_mean": 0.0005789952119812369,
+      "k1_gt_raw_min": -0.004300437867641449,
+      "k1_gt_raw_neg_pct": 5.8451714515686035,
+      "k1_sample_max": -0.09452944993972778,
+      "k1_sample_mean": -0.1174035519361496,
+      "k1_sample_min": -0.1418294906616211,
+      "k1_sample_neg_pct": 8.4375,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.005076252669095993,
+      "mask3_pct": 0.12536077201366425,
+      "neg_student_nll_loss": 0.0031956052407622337,
+      "sampled_diff_pct": 0.129419207572937,
+      "step": 920
+    },
+    {
+      "adv1_neg_pct": 6.187288284301758,
+      "adv2_active_pct": 0.08470146358013153,
+      "ce_loss": 0.001648780656978488,
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.1643683910369873,
+      "hpd_loss": 0.0010852484265342355,
+      "k1_gt_raw_max": 0.03225347399711609,
+      "k1_gt_raw_mean": 0.0005872219335287809,
+      "k1_gt_raw_min": -0.0068657128140330315,
+      "k1_gt_raw_neg_pct": 6.187288284301758,
+      "k1_sample_max": -0.11387737095355988,
+      "k1_sample_mean": -0.13910287618637085,
+      "k1_sample_min": -0.1643284261226654,
+      "k1_sample_neg_pct": 7.8125,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.0010852484032511712,
+      "mask3_pct": 0.08470146358013153,
+      "neg_student_nll_loss": 0.001648780656978488,
+      "sampled_diff_pct": 0.08656158298254013,
+      "step": 940
+    },
+    {
+      "adv1_neg_pct": 5.000968933105469,
+      "adv2_active_pct": 0.110027015209198,
+      "ce_loss": 0.001415811711922288,
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.6776,
+      "eval_generator_rouge1": 99.8328,
+      "eval_generator_rouge2": 99.7335,
+      "eval_generator_rougeL": 99.6469,
+      "eval_loss": 0.00400863541290164,
+      "eval_runtime": 591.8273,
+      "eval_samples_per_second": 5.769,
+      "eval_selector_accuracy": 97.7804,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.181,
+      "hpd_loss": 0.0011558050755411386,
+      "k1_gt_raw_max": 0.03289224952459335,
+      "k1_gt_raw_mean": 0.0007148425211198628,
+      "k1_gt_raw_min": -0.0005029054591432214,
+      "k1_gt_raw_neg_pct": 5.000969409942627,
+      "k1_sample_max": -0.4340977072715759,
+      "k1_sample_mean": -0.4340977072715759,
+      "k1_sample_min": -0.4340977072715759,
+      "k1_sample_neg_pct": 9.090909004211426,
+      "mask3_pct": 0.110027015209198,
+      "neg_student_nll_loss": 0.001415811711922288,
+      "sampled_diff_pct": 0.110027015209198,
+      "step": 942
+    },
+    {
+      "adv1_neg_pct": 5.784729957580566,
+      "adv2_active_pct": 0.1198313981294632,
+      "ce_loss": 0.0024845735169947147,
+      "epoch": 3.057439170323095,
+      "grad_norm": 0.0822562500834465,
+      "hpd_loss": 0.0026429281570017338,
+      "k1_gt_raw_max": 0.030722517520189285,
+      "k1_gt_raw_mean": 0.0009094014531001449,
+      "k1_gt_raw_min": -0.006959723308682442,
+      "k1_gt_raw_neg_pct": 5.784729957580566,
+      "k1_sample_max": -0.06599755585193634,
+      "k1_sample_mean": -0.06599755585193634,
+      "k1_sample_min": -0.06599755585193634,
+      "k1_sample_neg_pct": 6.94444465637207,
+      "learning_rate": 3.3007844105971336e-05,
+      "loss": 0.0024656012654304506,
+      "mask3_pct": 0.1198313981294632,
+      "neg_student_nll_loss": 0.0024845735169947147,
+      "sampled_diff_pct": 0.12411808967590332,
+      "step": 960
+    },
+    {
+      "adv1_neg_pct": 6.275859832763672,
+      "adv2_active_pct": 0.1653088629245758,
+      "ce_loss": 0.002910045674070716,
+      "epoch": 3.12126047068209,
+      "grad_norm": 0.002782815834507346,
+      "hpd_loss": 0.002390575595200062,
+      "k1_gt_raw_max": 0.03876848146319389,
+      "k1_gt_raw_mean": 0.001082484028302133,
+      "k1_gt_raw_min": -0.005310694221407175,
+      "k1_gt_raw_neg_pct": 6.275859832763672,
+      "k1_sample_max": -0.10106666386127472,
+      "k1_sample_mean": -0.10889029502868652,
+      "k1_sample_min": -0.11623653769493103,
+      "k1_sample_neg_pct": 7.8125,
+      "learning_rate": 3.114292593987149e-05,
+      "loss": 0.0023905765265226364,
+      "mask3_pct": 0.1653088629245758,
+      "neg_student_nll_loss": 0.002910045674070716,
+      "sampled_diff_pct": 0.17157939076423645,
+      "step": 980
+    },
+    {
+      "adv1_neg_pct": 6.713335037231445,
+      "adv2_active_pct": 0.07989098131656647,
+      "ce_loss": 0.000890995841473341,
+      "epoch": 3.185081771041085,
+      "grad_norm": 0.025590306147933006,
+      "hpd_loss": 0.0010361053282395005,
+      "k1_gt_raw_max": 0.02309339866042137,
+      "k1_gt_raw_mean": 0.000387923966627568,
+      "k1_gt_raw_min": -0.0018807905726134777,
+      "k1_gt_raw_neg_pct": 6.713335037231445,
+      "k1_sample_max": -0.07003647089004517,
+      "k1_sample_mean": -0.0700383111834526,
+      "k1_sample_min": -0.07004015147686005,
+      "k1_sample_neg_pct": 6.5625,
+      "learning_rate": 2.9308285746383408e-05,
+      "loss": 0.0010361054912209511,
+      "mask3_pct": 0.07989098131656647,
+      "neg_student_nll_loss": 0.000890995841473341,
+      "sampled_diff_pct": 0.08394942432641983,
+      "step": 1000
+    },
+    {
+      "adv1_neg_pct": 6.980464458465576,
+      "adv2_active_pct": 0.04861818999052048,
+      "ce_loss": 0.0008955475641414523,
+      "epoch": 3.24890307140008,
+      "grad_norm": 0.04064895957708359,
+      "hpd_loss": 0.0005541563150472939,
+      "k1_gt_raw_max": 0.023353416472673416,
+      "k1_gt_raw_mean": 0.00037372263614088297,
+      "k1_gt_raw_min": -0.0031757489778101444,
+      "k1_gt_raw_neg_pct": 6.980464458465576,
+      "k1_sample_max": -0.02265203185379505,
+      "k1_sample_mean": -0.02265203185379505,
+      "k1_sample_min": -0.02265203185379505,
+      "k1_sample_neg_pct": 5.625,
+      "learning_rate": 2.7506861536187867e-05,
+      "loss": 0.0005541563965380192,
+      "mask3_pct": 0.04861818999052048,
+      "neg_student_nll_loss": 0.0008955475641414523,
+      "sampled_diff_pct": 0.04861818999052048,
+      "step": 1020
+    },
+    {
+      "adv1_neg_pct": 6.828683853149414,
+      "adv2_active_pct": 0.0889447033405304,
+      "ce_loss": 0.0019629367161542177,
+      "epoch": 3.3127243717590744,
+      "grad_norm": 2.8549108505249023,
+      "hpd_loss": 0.002801127266138792,
+      "k1_gt_raw_max": 0.021204540506005287,
+      "k1_gt_raw_mean": 0.00043139539775438607,
+      "k1_gt_raw_min": -0.006724623031914234,
+      "k1_gt_raw_neg_pct": 6.828684329986572,
+      "k1_sample_max": -0.07739919424057007,
+      "k1_sample_mean": -0.07739919424057007,
+      "k1_sample_min": -0.07739919424057007,
+      "k1_sample_neg_pct": 5.0,
+      "learning_rate": 2.5741538127566396e-05,
+      "loss": 0.00280112698674202,
+      "mask3_pct": 0.0889447033405304,
+      "neg_student_nll_loss": 0.0019629367161542177,
+      "sampled_diff_pct": 0.10269542038440704,
+      "step": 1040
+    },
+    {
+      "adv1_neg_pct": 7.2235517501831055,
+      "adv2_active_pct": 0.10061224550008774,
+      "ce_loss": 0.004100818186998367,
+      "epoch": 3.3765456721180693,
+      "grad_norm": 0.004515279550105333,
+      "hpd_loss": 0.006908780429512262,
+      "k1_gt_raw_max": 0.017903421074151993,
+      "k1_gt_raw_mean": 0.0002864316338673234,
+      "k1_gt_raw_min": -0.003864757716655731,
+      "k1_gt_raw_neg_pct": 7.2235517501831055,
+      "k1_sample_max": -0.07832760363817215,
+      "k1_sample_mean": -0.07832760363817215,
+      "k1_sample_min": -0.07832760363817215,
+      "k1_sample_neg_pct": 5.9375,
+      "learning_rate": 2.4015142526625115e-05,
+      "loss": 0.006908781081438065,
+      "mask3_pct": 0.10061224550008774,
+      "neg_student_nll_loss": 0.004100818186998367,
+      "sampled_diff_pct": 0.10612311214208603,
+      "step": 1060
+    },
+    {
+      "adv1_neg_pct": 7.121249198913574,
+      "adv2_active_pct": 0.035785384476184845,
+      "ce_loss": 0.0010942043736577034,
+      "epoch": 3.4403669724770642,
+      "grad_norm": 0.006863700225949287,
+      "hpd_loss": 0.0009617547038942575,
+      "k1_gt_raw_max": 0.016735289245843887,
+      "k1_gt_raw_mean": 0.0003524690982885659,
+      "k1_gt_raw_min": -0.003919572569429874,
+      "k1_gt_raw_neg_pct": 7.121249198913574,
+      "k1_sample_max": -0.02336699143052101,
+      "k1_sample_mean": -0.02336699143052101,
+      "k1_sample_min": -0.02336699143052101,
+      "k1_sample_neg_pct": 4.0625,
+      "learning_rate": 2.233043940009967e-05,
+      "loss": 0.0009617548435926438,
+      "mask3_pct": 0.032108910381793976,
+      "neg_student_nll_loss": 0.0010942043736577034,
+      "sampled_diff_pct": 0.05000416934490204,
+      "step": 1080
+    },
+    {
+      "adv1_neg_pct": 7.497800827026367,
+      "adv2_active_pct": 0.06837265938520432,
+      "ce_loss": 0.0012112363474443555,
+      "epoch": 3.504188272836059,
+      "grad_norm": 0.009582476690411568,
+      "hpd_loss": 0.0011069111060351133,
+      "k1_gt_raw_max": 0.01962250843644142,
+      "k1_gt_raw_mean": 0.0003861070144921541,
+      "k1_gt_raw_min": -0.003895876696333289,
+      "k1_gt_raw_neg_pct": 7.497800827026367,
+      "k1_sample_max": -0.03470407798886299,
+      "k1_sample_mean": -0.038440171629190445,
+      "k1_sample_min": -0.0421762689948082,
+      "k1_sample_neg_pct": 5.78125,
+      "learning_rate": 2.0690126647990974e-05,
+      "loss": 0.0011069110594689846,
+      "mask3_pct": 0.06837265938520432,
+      "neg_student_nll_loss": 0.0012112363474443555,
+      "sampled_diff_pct": 0.07695083320140839,
+      "step": 1100
+    },
+    {
+      "adv1_neg_pct": 7.4989423751831055,
+      "adv2_active_pct": 0.09932753443717957,
+      "ce_loss": 0.001159896608442068,
+      "epoch": 3.568009573195054,
+      "grad_norm": 0.07570958882570267,
+      "hpd_loss": 0.0004915354074910283,
+      "k1_gt_raw_max": 0.021993957459926605,
+      "k1_gt_raw_mean": 0.0004521916271187365,
+      "k1_gt_raw_min": -0.005119780544191599,
+      "k1_gt_raw_neg_pct": 7.4989423751831055,
+      "k1_sample_max": -0.03237403556704521,
+      "k1_sample_mean": -0.06356782466173172,
+      "k1_sample_min": -0.09476162493228912,
+      "k1_sample_neg_pct": 6.5625,
+      "learning_rate": 1.9096831083121828e-05,
+      "loss": 0.0004915353842079639,
+      "mask3_pct": 0.09932753443717957,
+      "neg_student_nll_loss": 0.001159896608442068,
+      "sampled_diff_pct": 0.10849244892597198,
+      "step": 1120
+    },
+    {
+      "adv1_neg_pct": 7.469758033752441,
+      "adv2_active_pct": 0.03868325054645538,
+      "ce_loss": 0.0013367629144340754,
+      "epoch": 3.6318308735540485,
+      "grad_norm": 0.030605977401137352,
+      "hpd_loss": 0.0013018195750191808,
+      "k1_gt_raw_max": 0.01742619276046753,
+      "k1_gt_raw_mean": 0.00042788474820554256,
+      "k1_gt_raw_min": -0.002206374891102314,
+      "k1_gt_raw_neg_pct": 7.469758033752441,
+      "k1_sample_max": -0.026432175189256668,
+      "k1_sample_mean": -0.026432175189256668,
+      "k1_sample_min": -0.026432175189256668,
+      "k1_sample_neg_pct": 4.375,
+      "learning_rate": 1.7553104224533324e-05,
+      "loss": 0.0013018193654716014,
+      "mask3_pct": 0.036279406398534775,
+      "neg_student_nll_loss": 0.0013367629144340754,
+      "sampled_diff_pct": 0.04094774276018143,
+      "step": 1140
+    },
+    {
+      "adv1_neg_pct": 7.294529914855957,
+      "adv2_active_pct": 0.11667037010192871,
+      "ce_loss": 0.0022262034472078085,
+      "epoch": 3.6956521739130435,
+      "grad_norm": 0.1906837671995163,
+      "hpd_loss": 0.0014206715859472752,
+      "k1_gt_raw_max": 0.021399978548288345,
+      "k1_gt_raw_mean": 0.00044373038690537214,
+      "k1_gt_raw_min": -0.0028102025389671326,
+      "k1_gt_raw_neg_pct": 7.294529914855957,
+      "k1_sample_max": -0.02788049727678299,
+      "k1_sample_mean": -0.028852656483650208,
+      "k1_sample_min": -0.03075457364320755,
+      "k1_sample_neg_pct": 4.9479169845581055,
+      "learning_rate": 1.606141821145735e-05,
+      "loss": 0.0014206715859472752,
+      "mask3_pct": 0.11374981701374054,
+      "neg_student_nll_loss": 0.0022262034472078085,
+      "sampled_diff_pct": 0.1271422654390335,
+      "step": 1160
+    },
+    {
+      "adv1_neg_pct": 7.579638957977295,
+      "adv2_active_pct": 0.06752925366163254,
+      "ce_loss": 0.0028973212465643883,
+      "epoch": 3.7594734742720384,
+      "grad_norm": 0.011799277737736702,
+      "hpd_loss": 0.0015698154456913471,
+      "k1_gt_raw_max": 0.01623934879899025,
+      "k1_gt_raw_mean": 0.00032681378070265055,
+      "k1_gt_raw_min": -0.0041321092285215855,
+      "k1_gt_raw_neg_pct": 7.579638957977295,
+      "k1_sample_max": -0.0820823684334755,
+      "k1_sample_mean": -0.08302680402994156,
+      "k1_sample_min": -0.08577128499746323,
+      "k1_sample_neg_pct": 3.680555582046509,
+      "learning_rate": 1.4624161844408763e-05,
+      "loss": 0.0015698155388236047,
+      "mask3_pct": 0.06483529508113861,
+      "neg_student_nll_loss": 0.0028973212465643883,
+      "sampled_diff_pct": 0.08027052134275436,
+      "step": 1180
+    },
+    {
+      "adv1_neg_pct": 8.004940032958984,
+      "adv2_active_pct": 0.17861586809158325,
+      "ce_loss": 0.0022735765669494867,
+      "epoch": 3.823294774631033,
+      "grad_norm": 0.014485353603959084,
+      "hpd_loss": 0.0013695353409275413,
+      "k1_gt_raw_max": 0.022271674126386642,
+      "k1_gt_raw_mean": 0.00047311594244092703,
+      "k1_gt_raw_min": -0.005409082397818565,
+      "k1_gt_raw_neg_pct": 8.004940032958984,
+      "k1_sample_max": -0.07869651913642883,
+      "k1_sample_mean": -0.1084669977426529,
+      "k1_sample_min": -0.13823749125003815,
+      "k1_sample_neg_pct": 6.5625,
+      "learning_rate": 1.324363675973698e-05,
+      "loss": 0.0013695357367396355,
+      "mask3_pct": 0.17861586809158325,
+      "neg_student_nll_loss": 0.0022735765669494867,
+      "sampled_diff_pct": 0.1825408637523651,
+      "step": 1200
+    },
+    {
+      "adv1_neg_pct": 7.9708051681518555,
+      "adv2_active_pct": 0.14446821808815002,
+      "ce_loss": 0.002031442243605852,
+      "epoch": 3.8871160749900278,
+      "grad_norm": 1.8391499519348145,
+      "hpd_loss": 0.0014964366564527154,
+      "k1_gt_raw_max": 0.019910190254449844,
+      "k1_gt_raw_mean": 0.0005614210967905819,
+      "k1_gt_raw_min": -0.003311981214210391,
+      "k1_gt_raw_neg_pct": 7.9708051681518555,
+      "k1_sample_max": -0.06566861271858215,
+      "k1_sample_mean": -0.06566861271858215,
+      "k1_sample_min": -0.06566861271858215,
+      "k1_sample_neg_pct": 5.3125,
+      "learning_rate": 1.1922053743763052e-05,
+      "loss": 0.0014964366331696511,
+      "mask3_pct": 0.14446821808815002,
+      "neg_student_nll_loss": 0.002031442243605852,
+      "sampled_diff_pct": 0.1486356258392334,
+      "step": 1220
+    },
+    {
+      "adv1_neg_pct": 7.752712249755859,
+      "adv2_active_pct": 0.05451521277427673,
+      "ce_loss": 0.0008983995649032295,
+      "epoch": 3.9509373753490227,
+      "grad_norm": 0.015798989683389664,
+      "hpd_loss": 0.0006963292835280299,
+      "k1_gt_raw_max": 0.017497122287750244,
+      "k1_gt_raw_mean": 0.00027755158953368664,
+      "k1_gt_raw_min": -0.0025472629349678755,
+      "k1_gt_raw_neg_pct": 7.752712249755859,
+      "k1_sample_max": -0.0381598025560379,
+      "k1_sample_mean": -0.03892132267355919,
+      "k1_sample_min": -0.03968284651637077,
+      "k1_sample_neg_pct": 4.375,
+      "learning_rate": 1.0661529192404973e-05,
+      "loss": 0.0006963292602449655,
+      "mask3_pct": 0.05451521277427673,
+      "neg_student_nll_loss": 0.0008983995649032295,
+      "sampled_diff_pct": 0.061319805681705475,
+      "step": 1240
+    },
+    {
+      "adv1_neg_pct": 7.9339280128479,
+      "adv2_active_pct": 0.0740404948592186,
+      "ce_loss": 0.0014802285004407167,
+      "epoch": 4.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.4953,
+      "eval_generator_rouge1": 99.8312,
+      "eval_generator_rouge2": 99.7559,
+      "eval_generator_rougeL": 99.645,
+      "eval_loss": 0.003369994228705764,
+      "eval_runtime": 1103.3525,
+      "eval_samples_per_second": 3.094,
+      "eval_selector_accuracy": 98.5981,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.097,
+      "hpd_loss": 0.0012612877180799842,
+      "k1_gt_raw_max": 0.01940547674894333,
+      "k1_gt_raw_mean": 0.0002844607806764543,
+      "k1_gt_raw_min": -0.003394282190129161,
+      "k1_gt_raw_neg_pct": 7.9339280128479,
+      "k1_sample_max": -0.06970421969890594,
+      "k1_sample_mean": -0.06970421969890594,
+      "k1_sample_min": -0.06970421969890594,
+      "k1_sample_neg_pct": 6.910569190979004,
+      "mask3_pct": 0.0740404948592186,
+      "neg_student_nll_loss": 0.0014802285004407167,
+      "sampled_diff_pct": 0.08771276473999023,
+      "step": 1256
+    },
+    {
+      "adv1_neg_pct": 8.280838966369629,
+      "adv2_active_pct": 0.13949164748191833,
+      "ce_loss": 0.005130399018526077,
+      "epoch": 4.0127642600717985,
+      "grad_norm": 0.059252846986055374,
+      "hpd_loss": 0.003739401465281844,
+      "k1_gt_raw_max": 0.02487422712147236,
+      "k1_gt_raw_mean": 0.00037547305691987276,
+      "k1_gt_raw_min": -0.009174536913633347,
+      "k1_gt_raw_neg_pct": 8.280838012695312,
+      "k1_sample_max": -0.009557662531733513,
+      "k1_sample_mean": -0.013778358697891235,
+      "k1_sample_min": -0.020752470940351486,
+      "k1_sample_neg_pct": 7.5,
+      "learning_rate": 9.464081721960592e-06,
+      "loss": 0.001723523810505867,
+      "mask3_pct": 0.13949164748191833,
+      "neg_student_nll_loss": 0.005130399018526077,
+      "sampled_diff_pct": 0.17727938294410706,
+      "step": 1260
+    },
+    {
+      "adv1_neg_pct": 7.800583839416504,
+      "adv2_active_pct": 0.043229397386312485,
+      "ce_loss": 0.0006054828409105539,
+      "epoch": 4.076585560430794,
+      "grad_norm": 0.0579792857170105,
+      "hpd_loss": 0.0004133617621846497,
+      "k1_gt_raw_max": 0.015528247691690922,
+      "k1_gt_raw_mean": 0.0002333328011445701,
+      "k1_gt_raw_min": -0.0038630864582955837,
+      "k1_gt_raw_neg_pct": 7.800584316253662,
+      "k1_sample_max": -0.014678173698484898,
+      "k1_sample_mean": -0.014678173698484898,
+      "k1_sample_min": -0.014678173698484898,
+      "k1_sample_neg_pct": 4.0625,
+      "learning_rate": 8.331628936475818e-06,
+      "loss": 0.0004133617505431175,
+      "mask3_pct": 0.043229397386312485,
+      "neg_student_nll_loss": 0.0006054828409105539,
+      "sampled_diff_pct": 0.045146577060222626,
+      "step": 1280
+    },
+    {
+      "adv1_neg_pct": 8.047872543334961,
+      "adv2_active_pct": 0.08839298039674759,
+      "ce_loss": 0.0009541312465444207,
+      "epoch": 4.140406860789788,
+      "grad_norm": 0.003797554410994053,
+      "hpd_loss": 0.0009471764788031578,
+      "k1_gt_raw_max": 0.013323026709258556,
+      "k1_gt_raw_mean": 0.00020666152704507113,
+      "k1_gt_raw_min": -0.00606294721364975,
+      "k1_gt_raw_neg_pct": 8.047872543334961,
+      "k1_sample_max": -0.008535138331353664,
+      "k1_sample_mean": -0.008535138331353664,
+      "k1_sample_min": -0.008535138331353664,
+      "k1_sample_neg_pct": 5.3125,
+      "learning_rate": 7.2659843568748385e-06,
+      "loss": 0.0009471764788031578,
+      "mask3_pct": 0.08839298039674759,
+      "neg_student_nll_loss": 0.0009541312465444207,
+      "sampled_diff_pct": 0.09454742819070816,
+      "step": 1300
+    },
+    {
+      "adv1_neg_pct": 8.000151634216309,
+      "adv2_active_pct": 0.034580253064632416,
+      "ce_loss": 0.0007429468678310513,
+      "epoch": 4.204228161148784,
+      "grad_norm": 0.022960586473345757,
+      "hpd_loss": 0.0005513728247024119,
+      "k1_gt_raw_max": 0.015379039570689201,
+      "k1_gt_raw_mean": 0.0002641355968080461,
+      "k1_gt_raw_min": -0.002805485622957349,
+      "k1_gt_raw_neg_pct": 8.000151634216309,
+      "k1_sample_max": -0.006287828553467989,
+      "k1_sample_mean": -0.006287828553467989,
+      "k1_sample_min": -0.006287828553467989,
+      "k1_sample_neg_pct": 3.4375,
+      "learning_rate": 6.268854516770008e-06,
+      "loss": 0.000551372766494751,
+      "mask3_pct": 0.034580253064632416,
+      "neg_student_nll_loss": 0.0007429468678310513,
+      "sampled_diff_pct": 0.047987110912799835,
+      "step": 1320
+    },
+    {
+      "adv1_neg_pct": 8.06116008758545,
+      "adv2_active_pct": 0.02886439487338066,
+      "ce_loss": 0.0009998048190027475,
+      "epoch": 4.268049461507778,
+      "grad_norm": 0.05514178052544594,
+      "hpd_loss": 0.0010553341126069427,
+      "k1_gt_raw_max": 0.014247752726078033,
+      "k1_gt_raw_mean": 0.0003266474523115903,
+      "k1_gt_raw_min": -0.0025498915929347277,
+      "k1_gt_raw_neg_pct": 8.06116008758545,
+      "k1_sample_max": -0.023031100630760193,
+      "k1_sample_mean": -0.023031100630760193,
+      "k1_sample_min": -0.023031100630760193,
+      "k1_sample_neg_pct": 3.4375,
+      "learning_rate": 5.341836229602331e-06,
+      "loss": 0.0010553342290222645,
+      "mask3_pct": 0.02886439487338066,
+      "neg_student_nll_loss": 0.0009998048190027475,
+      "sampled_diff_pct": 0.03517622873187065,
+      "step": 1340
+    },
+    {
+      "adv1_neg_pct": 8.273700714111328,
+      "adv2_active_pct": 0.022702474147081375,
+      "ce_loss": 0.0009629722917452455,
+      "epoch": 4.331870761866773,
+      "grad_norm": 0.02074464224278927,
+      "hpd_loss": 0.0008157925913110375,
+      "k1_gt_raw_max": 0.01817188784480095,
+      "k1_gt_raw_mean": 0.0004169899912085384,
+      "k1_gt_raw_min": -0.003591842483729124,
+      "k1_gt_raw_neg_pct": 8.273700714111328,
+      "k1_sample_max": -0.006041577085852623,
+      "k1_sample_mean": -0.006041577085852623,
+      "k1_sample_min": -0.006041577085852623,
+      "k1_sample_neg_pct": 2.8125,
+      "learning_rate": 4.4864140314888216e-06,
+      "loss": 0.0008157927542924881,
+      "mask3_pct": 0.022702474147081375,
+      "neg_student_nll_loss": 0.0009629722917452455,
+      "sampled_diff_pct": 0.030936820432543755,
+      "step": 1360
+    },
+    {
+      "adv1_neg_pct": 7.839452743530273,
+      "adv2_active_pct": 0.049231112003326416,
+      "ce_loss": 0.0010351771488785744,
+      "epoch": 4.395692062225768,
+      "grad_norm": 0.038118693977594376,
+      "hpd_loss": 0.0007022822974249721,
+      "k1_gt_raw_max": 0.013979194685816765,
+      "k1_gt_raw_mean": 0.00021736414055339992,
+      "k1_gt_raw_min": -0.005359914153814316,
+      "k1_gt_raw_neg_pct": 7.839452743530273,
+      "k1_sample_max": -0.04112572968006134,
+      "k1_sample_mean": -0.04112572968006134,
+      "k1_sample_min": -0.04112572968006134,
+      "k1_sample_neg_pct": 4.6875,
+      "learning_rate": 3.703957803871932e-06,
+      "loss": 0.0007022821810096502,
+      "mask3_pct": 0.049231112003326416,
+      "neg_student_nll_loss": 0.0010351771488785744,
+      "sampled_diff_pct": 0.056373219937086105,
+      "step": 1380
+    },
+    {
+      "adv1_neg_pct": 8.548715591430664,
+      "adv2_active_pct": 0.06189306825399399,
+      "ce_loss": 0.002533749444410205,
+      "epoch": 4.4595133625847625,
+      "grad_norm": 0.04451121762394905,
+      "hpd_loss": 0.0015958126168698072,
+      "k1_gt_raw_max": 0.017123829573392868,
+      "k1_gt_raw_mean": 0.0002977260737679899,
+      "k1_gt_raw_min": -0.0033434557262808084,
+      "k1_gt_raw_neg_pct": 8.548715591430664,
+      "k1_sample_max": -0.018765060231089592,
+      "k1_sample_mean": -0.02024500072002411,
+      "k1_sample_min": -0.022792842239141464,
+      "k1_sample_neg_pct": 4.699519157409668,
+      "learning_rate": 2.9957205797780495e-06,
+      "loss": 0.0015958130359649658,
+      "mask3_pct": 0.05997588858008385,
+      "neg_student_nll_loss": 0.002533749444410205,
+      "sampled_diff_pct": 0.07589613646268845,
+      "step": 1400
+    },
+    {
+      "adv1_neg_pct": 8.14011001586914,
+      "adv2_active_pct": 0.028554929420351982,
+      "ce_loss": 0.0016532940790057182,
+      "epoch": 4.523334662943758,
+      "grad_norm": 0.03609246015548706,
+      "hpd_loss": 0.0018588351085782051,
+      "k1_gt_raw_max": 0.015877414494752884,
+      "k1_gt_raw_mean": 0.0004283803573343903,
+      "k1_gt_raw_min": -0.004388327710330486,
+      "k1_gt_raw_neg_pct": 8.14011001586914,
+      "k1_sample_max": -0.019240127876400948,
+      "k1_sample_mean": -0.019240127876400948,
+      "k1_sample_min": -0.019240127876400948,
+      "k1_sample_neg_pct": 3.125,
+      "learning_rate": 2.362836537198199e-06,
+      "loss": 0.0018588351085782051,
+      "mask3_pct": 0.028554929420351982,
+      "neg_student_nll_loss": 0.0016532940790057182,
+      "sampled_diff_pct": 0.041689325124025345,
+      "step": 1420
+    },
+    {
+      "adv1_neg_pct": 8.328222274780273,
+      "adv2_active_pct": 0.050987400114536285,
+      "ce_loss": 0.0007530204020440578,
+      "epoch": 4.587155963302752,
+      "grad_norm": 0.01626456528902054,
+      "hpd_loss": 0.0006849722703918815,
+      "k1_gt_raw_max": 0.014860251918435097,
+      "k1_gt_raw_mean": 0.0002935512748081237,
+      "k1_gt_raw_min": -0.003724183887243271,
+      "k1_gt_raw_neg_pct": 8.328222274780273,
+      "k1_sample_max": -0.028437752276659012,
+      "k1_sample_mean": -0.028437752276659012,
+      "k1_sample_min": -0.028437752276659012,
+      "k1_sample_neg_pct": 4.6875,
+      "learning_rate": 1.8063191828043703e-06,
+      "loss": 0.0006849721074104309,
+      "mask3_pct": 0.050987400114536285,
+      "neg_student_nll_loss": 0.0007530204020440578,
+      "sampled_diff_pct": 0.06721149384975433,
+      "step": 1440
+    },
+    {
+      "adv1_neg_pct": 7.96031379699707,
+      "adv2_active_pct": 0.03354815021157265,
+      "ce_loss": 0.0007837152807042003,
+      "epoch": 4.650977263661747,
+      "grad_norm": 0.7522907257080078,
+      "hpd_loss": 0.0007729316130280495,
+      "k1_gt_raw_max": 0.014540698379278183,
+      "k1_gt_raw_mean": 0.00021094034309498966,
+      "k1_gt_raw_min": -0.003531917929649353,
+      "k1_gt_raw_neg_pct": 7.96031379699707,
+      "k1_sample_max": -0.007455674931406975,
+      "k1_sample_mean": -0.036636389791965485,
+      "k1_sample_min": -0.06581709533929825,
+      "k1_sample_neg_pct": 3.28125,
+      "learning_rate": 1.3270597289100437e-06,
+      "loss": 0.000772931519895792,
+      "mask3_pct": 0.03354815021157265,
+      "neg_student_nll_loss": 0.0007837152807042003,
+      "sampled_diff_pct": 0.04775560647249222,
+      "step": 1460
+    },
+    {
+      "adv1_neg_pct": 8.403984069824219,
+      "adv2_active_pct": 0.02325102873146534,
+      "ce_loss": 0.0005917788948863745,
+      "epoch": 4.714798564020742,
+      "grad_norm": 0.0069316113367676735,
+      "hpd_loss": 0.000479920010548085,
+      "k1_gt_raw_max": 0.01145990937948227,
+      "k1_gt_raw_mean": 0.0001835223811212927,
+      "k1_gt_raw_min": -0.004522635601460934,
+      "k1_gt_raw_neg_pct": 8.403984069824219,
+      "k1_sample_max": -0.0059456657618284225,
+      "k1_sample_mean": -0.008637822233140469,
+      "k1_sample_min": -0.012212925590574741,
+      "k1_sample_neg_pct": 2.7083334922790527,
+      "learning_rate": 9.258256662741205e-07,
+      "loss": 0.00047991983592510223,
+      "mask3_pct": 0.02325102873146534,
+      "neg_student_nll_loss": 0.0005917788948863745,
+      "sampled_diff_pct": 0.029702603816986084,
+      "step": 1480
+    },
+    {
+      "adv1_neg_pct": 7.90092658996582,
+      "adv2_active_pct": 0.05479162558913231,
+      "ce_loss": 0.0017136812675744295,
+      "epoch": 4.778619864379737,
+      "grad_norm": 0.08339332044124603,
+      "hpd_loss": 0.0012177170719951391,
+      "k1_gt_raw_max": 0.015806099399924278,
+      "k1_gt_raw_mean": 0.00033896442619152367,
+      "k1_gt_raw_min": -0.0018794997595250607,
+      "k1_gt_raw_neg_pct": 7.90092658996582,
+      "k1_sample_max": -0.02365684136748314,
+      "k1_sample_mean": -0.02365684136748314,
+      "k1_sample_min": -0.02365684136748314,
+      "k1_sample_neg_pct": 5.9375,
+      "learning_rate": 6.032595350337341e-07,
+      "loss": 0.001217717118561268,
+      "mask3_pct": 0.05479162558913231,
+      "neg_student_nll_loss": 0.0017136812675744295,
+      "sampled_diff_pct": 0.0594295933842659,
+      "step": 1500
+    },
+    {
+      "adv1_neg_pct": 8.259902954101562,
+      "adv2_active_pct": 0.09987135231494904,
+      "ce_loss": 0.004173032008111477,
+      "epoch": 4.842441164738732,
+      "grad_norm": 0.010871863923966885,
+      "hpd_loss": 0.007525380700826645,
+      "k1_gt_raw_max": 0.01568448543548584,
+      "k1_gt_raw_mean": 0.0002367629058426246,
+      "k1_gt_raw_min": -0.0033387483563274145,
+      "k1_gt_raw_neg_pct": 8.259902954101562,
+      "k1_sample_max": -0.04992266744375229,
+      "k1_sample_mean": -0.05137576535344124,
+      "k1_sample_min": -0.05282886326313019,
+      "k1_sample_neg_pct": 5.625,
+      "learning_rate": 3.5987789573421356e-07,
+      "loss": 0.007525377720594406,
+      "mask3_pct": 0.09987135231494904,
+      "neg_student_nll_loss": 0.004173032008111477,
+      "sampled_diff_pct": 0.11135074496269226,
+      "step": 1520
+    },
+    {
+      "adv1_neg_pct": 8.341958045959473,
+      "adv2_active_pct": 0.044122375547885895,
+      "ce_loss": 0.0007239175029098988,
+      "epoch": 4.9062624650977265,
+      "grad_norm": 0.09346315264701843,
+      "hpd_loss": 0.0006393846706487238,
+      "k1_gt_raw_max": 0.012377542443573475,
+      "k1_gt_raw_mean": 0.0001652358623687178,
+      "k1_gt_raw_min": -0.004152372479438782,
+      "k1_gt_raw_neg_pct": 8.341958999633789,
+      "k1_sample_max": -0.007577377371490002,
+      "k1_sample_mean": -0.009641861543059349,
+      "k1_sample_min": -0.012904488481581211,
+      "k1_sample_neg_pct": 4.6875,
+      "learning_rate": 1.9607050210401192e-07,
+      "loss": 0.0006393846124410629,
+      "mask3_pct": 0.044122375547885895,
+      "neg_student_nll_loss": 0.0007239175029098988,
+      "sampled_diff_pct": 0.04648980498313904,
+      "step": 1540
+    },
+    {
+      "adv1_neg_pct": 8.45923137664795,
+      "adv2_active_pct": 0.06292080879211426,
+      "ce_loss": 0.0011851892340928316,
+      "epoch": 4.970083765456721,
+      "grad_norm": 0.028900589793920517,
+      "hpd_loss": 0.0013436225708574057,
+      "k1_gt_raw_max": 0.013746695592999458,
+      "k1_gt_raw_mean": 0.0003474469413049519,
+      "k1_gt_raw_min": -0.0036701993085443974,
+      "k1_gt_raw_neg_pct": 8.45923137664795,
+      "k1_sample_max": -0.009252920746803284,
+      "k1_sample_mean": -0.01030099019408226,
+      "k1_sample_min": -0.011349059641361237,
+      "k1_sample_neg_pct": 2.34375,
+      "learning_rate": 1.1209967689925832e-07,
+      "loss": 0.0013436229899525642,
+      "mask3_pct": 0.05779785290360451,
+      "neg_student_nll_loss": 0.0011851892340928316,
+      "sampled_diff_pct": 0.07132385671138763,
+      "step": 1560
+    },
+    {
+      "adv1_neg_pct": 8.752664566040039,
+      "adv2_active_pct": 0.050552137196063995,
+      "ce_loss": 0.0007130501326173544,
+      "epoch": 5.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 96.4953,
+      "eval_generator_rouge1": 99.897,
+      "eval_generator_rouge2": 99.8245,
+      "eval_generator_rougeL": 99.7185,
+      "eval_loss": 0.0032775094732642174,
+      "eval_runtime": 593.2024,
+      "eval_samples_per_second": 5.755,
+      "eval_selector_accuracy": 98.5981,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.18,
+      "hpd_loss": 0.0003927574143745005,
+      "k1_gt_raw_max": 0.015969211235642433,
+      "k1_gt_raw_mean": 0.00024475069949403405,
+      "k1_gt_raw_min": -0.00466952845454216,
+      "k1_gt_raw_neg_pct": 8.752664566040039,
+      "k1_sample_max": -0.008284520357847214,
+      "k1_sample_mean": -0.008284520357847214,
+      "k1_sample_min": -0.008284520357847214,
+      "k1_sample_neg_pct": 4.6666669845581055,
+      "mask3_pct": 0.050552137196063995,
+      "neg_student_nll_loss": 0.0007130501326173544,
+      "sampled_diff_pct": 0.061017539352178574,
+      "step": 1570
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.871444840310374e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/hpd/checkpoint-1570/training_args.bin b/qwen3/hpd/checkpoint-1570/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1caa0e72cc5edd590e105073725cf4d832684e44
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f04fac21a72f04221af020c3bb88a0dc0ccf53bb7a0c0da21d1ab444937cb9d
+size 7185
diff --git a/qwen3/hpd/checkpoint-1570/zero_to_fp32.py b/qwen3/hpd/checkpoint-1570/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/hpd/checkpoint-1570/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/hpd/checkpoint-314/README.md b/qwen3/hpd/checkpoint-314/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-314/adapter_config.json b/qwen3/hpd/checkpoint-314/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0f6594b49d741b9fe4eb4a1a4df0b797e15bcd
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-314/adapter_model.safetensors b/qwen3/hpd/checkpoint-314/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b6ff5631506ce4bc93b1784916220c3e279d752b
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1f90217ed788ded805849303db957d1cc8f5feb3f8187f49c8c1c9152bb7934
+size 40422560
diff --git a/qwen3/hpd/checkpoint-314/chat_template.jinja b/qwen3/hpd/checkpoint-314/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e80ef128bbd93e8bc5538556779deb458ab36a82
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1623d80d714a93923dd86442de63f5928b56cc71edeca1ae9b34b15343a6ed5
+size 121135557
diff --git a/qwen3/hpd/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..407f9d1cef9c445c342908e318c6e07894df462c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:457ae86f4d675b5bb0af89ec884c2f448a1a71ca6923d04c4d81ba7630993e8b
+size 121135749
diff --git a/qwen3/hpd/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/hpd/checkpoint-314/global_step314/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5acce95d0dd4016caccc6b70f1a36a18128f4e81
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/global_step314/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98cebb203413bed1ca3b57bb311570ab6933a02b1c092fd470e1da13546797cf
+size 351841765
diff --git a/qwen3/hpd/checkpoint-314/latest b/qwen3/hpd/checkpoint-314/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/latest
@@ -0,0 +1 @@
+global_step314
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-314/rng_state_0.pth b/qwen3/hpd/checkpoint-314/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..57cbbac1210236f19b1b59a3111389ffbc3eba31
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00eda5da8c40cf428aaf6fcef5640ebc27ef7abf4820654d2182813203b71aad
+size 14917
diff --git a/qwen3/hpd/checkpoint-314/rng_state_1.pth b/qwen3/hpd/checkpoint-314/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..402498990cb396eac6b683a31e8bb2dc2cf92b27
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b516d0c4aca53469f8a5924d3ddd531c3205f9f6c257531ee6c345db61620260
+size 14917
diff --git a/qwen3/hpd/checkpoint-314/scheduler.pt b/qwen3/hpd/checkpoint-314/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149
+size 1465
diff --git a/qwen3/hpd/checkpoint-314/tokenizer.json b/qwen3/hpd/checkpoint-314/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/hpd/checkpoint-314/tokenizer_config.json b/qwen3/hpd/checkpoint-314/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/hpd/checkpoint-314/trainer_state.json b/qwen3/hpd/checkpoint-314/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9101ac083aa5d2eb21d917d28c8709f998e3b4d5
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/trainer_state.json
@@ -0,0 +1,394 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 314,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "adv1_neg_pct": 0.2358350306749344,
+      "adv2_active_pct": 17.716575622558594,
+      "ce_loss": 0.6496895551681519,
+      "epoch": 0.06382130035899482,
+      "grad_norm": 4.9955339431762695,
+      "hpd_loss": 0.7524586915969849,
+      "k1_gt_raw_max": 0.3395599126815796,
+      "k1_gt_raw_mean": 0.08363302797079086,
+      "k1_gt_raw_min": -0.006161125842481852,
+      "k1_gt_raw_neg_pct": 0.2358350306749344,
+      "k1_sample_max": -1.2287571430206299,
+      "k1_sample_mean": -5.443553924560547,
+      "k1_sample_min": -17.516345977783203,
+      "k1_sample_neg_pct": 97.79017639160156,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.7524587631225585,
+      "mask3_pct": 17.716575622558594,
+      "neg_student_nll_loss": 0.6496895551681519,
+      "sampled_diff_pct": 17.718591690063477,
+      "step": 20
+    },
+    {
+      "adv1_neg_pct": 0.24569129943847656,
+      "adv2_active_pct": 6.35067081451416,
+      "ce_loss": 0.12904959917068481,
+      "epoch": 0.12764260071798963,
+      "grad_norm": 3.5179519653320312,
+      "hpd_loss": 0.07163389772176743,
+      "k1_gt_raw_max": 0.3387124538421631,
+      "k1_gt_raw_mean": 0.03725019469857216,
+      "k1_gt_raw_min": -0.009099474176764488,
+      "k1_gt_raw_neg_pct": 0.24569129943847656,
+      "k1_sample_max": -1.1939605474472046,
+      "k1_sample_mean": -4.561788082122803,
+      "k1_sample_min": -10.153608322143555,
+      "k1_sample_neg_pct": 96.875,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.07163387537002563,
+      "mask3_pct": 6.35067081451416,
+      "neg_student_nll_loss": 0.12904959917068481,
+      "sampled_diff_pct": 6.35067081451416,
+      "step": 40
+    },
+    {
+      "adv1_neg_pct": 0.18458101153373718,
+      "adv2_active_pct": 5.008382320404053,
+      "ce_loss": 0.10459579527378082,
+      "epoch": 0.19146390107698444,
+      "grad_norm": 2.890872001647949,
+      "hpd_loss": 0.09348365664482117,
+      "k1_gt_raw_max": 0.3109760880470276,
+      "k1_gt_raw_mean": 0.026082728058099747,
+      "k1_gt_raw_min": -0.01705370843410492,
+      "k1_gt_raw_neg_pct": 0.18458101153373718,
+      "k1_sample_max": -2.0660922527313232,
+      "k1_sample_mean": -4.749420166015625,
+      "k1_sample_min": -8.562150955200195,
+      "k1_sample_neg_pct": 89.6875,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.09348366260528565,
+      "mask3_pct": 5.008382320404053,
+      "neg_student_nll_loss": 0.10459579527378082,
+      "sampled_diff_pct": 5.008382320404053,
+      "step": 60
+    },
+    {
+      "adv1_neg_pct": 0.3009452819824219,
+      "adv2_active_pct": 3.9314260482788086,
+      "ce_loss": 0.08063317835330963,
+      "epoch": 0.25528520143597927,
+      "grad_norm": 2.033250570297241,
+      "hpd_loss": 0.08107596635818481,
+      "k1_gt_raw_max": 0.2881737947463989,
+      "k1_gt_raw_mean": 0.019415946677327156,
+      "k1_gt_raw_min": -0.008919830434024334,
+      "k1_gt_raw_neg_pct": 0.3009452819824219,
+      "k1_sample_max": -1.6159262657165527,
+      "k1_sample_mean": -3.7506155967712402,
+      "k1_sample_min": -6.360409259796143,
+      "k1_sample_neg_pct": 86.39583587646484,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.0810759723186493,
+      "mask3_pct": 3.9314260482788086,
+      "neg_student_nll_loss": 0.08063317835330963,
+      "sampled_diff_pct": 3.9378738403320312,
+      "step": 80
+    },
+    {
+      "adv1_neg_pct": 0.22911369800567627,
+      "adv2_active_pct": 2.694115161895752,
+      "ce_loss": 0.052447859197854996,
+      "epoch": 0.3191065017949741,
+      "grad_norm": 1.751448392868042,
+      "hpd_loss": 0.027988094836473465,
+      "k1_gt_raw_max": 0.3031776547431946,
+      "k1_gt_raw_mean": 0.01864592172205448,
+      "k1_gt_raw_min": -0.007533406838774681,
+      "k1_gt_raw_neg_pct": 0.22911369800567627,
+      "k1_sample_max": -1.57374906539917,
+      "k1_sample_mean": -3.1450629234313965,
+      "k1_sample_min": -5.145340919494629,
+      "k1_sample_neg_pct": 84.70052337646484,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.027988094091415405,
+      "mask3_pct": 2.6920177936553955,
+      "neg_student_nll_loss": 0.052447859197854996,
+      "sampled_diff_pct": 2.7128100395202637,
+      "step": 100
+    },
+    {
+      "adv1_neg_pct": 0.1952832043170929,
+      "adv2_active_pct": 3.306915044784546,
+      "ce_loss": 0.04980571195483208,
+      "epoch": 0.3829278021539689,
+      "grad_norm": 3.930035352706909,
+      "hpd_loss": 0.015151199884712696,
+      "k1_gt_raw_max": 0.28564560413360596,
+      "k1_gt_raw_mean": 0.01884515956044197,
+      "k1_gt_raw_min": -0.001684204675257206,
+      "k1_gt_raw_neg_pct": 0.1952832043170929,
+      "k1_sample_max": -1.9181127548217773,
+      "k1_sample_mean": -3.2553763389587402,
+      "k1_sample_min": -4.8449602127075195,
+      "k1_sample_neg_pct": 75.20833587646484,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.015151196718215942,
+      "mask3_pct": 3.306915044784546,
+      "neg_student_nll_loss": 0.04980571195483208,
+      "sampled_diff_pct": 3.3092129230499268,
+      "step": 120
+    },
+    {
+      "adv1_neg_pct": 0.3264433741569519,
+      "adv2_active_pct": 2.25433087348938,
+      "ce_loss": 0.04018855467438698,
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.6639359593391418,
+      "hpd_loss": 0.02619136869907379,
+      "k1_gt_raw_max": 0.24626323580741882,
+      "k1_gt_raw_mean": 0.015010751783847809,
+      "k1_gt_raw_min": -0.004701526835560799,
+      "k1_gt_raw_neg_pct": 0.3264433741569519,
+      "k1_sample_max": -1.902454137802124,
+      "k1_sample_mean": -2.805896282196045,
+      "k1_sample_min": -3.829012393951416,
+      "k1_sample_neg_pct": 71.25,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.026191365718841553,
+      "mask3_pct": 2.25433087348938,
+      "neg_student_nll_loss": 0.04018855467438698,
+      "sampled_diff_pct": 2.2565953731536865,
+      "step": 140
+    },
+    {
+      "adv1_neg_pct": 0.3137112259864807,
+      "adv2_active_pct": 1.3726557493209839,
+      "ce_loss": 0.03148110955953598,
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.548140525817871,
+      "hpd_loss": 0.03039727918803692,
+      "k1_gt_raw_max": 0.20344290137290955,
+      "k1_gt_raw_mean": 0.009032711386680603,
+      "k1_gt_raw_min": -0.0009857998229563236,
+      "k1_gt_raw_neg_pct": 0.3137112259864807,
+      "k1_sample_max": -2.153831958770752,
+      "k1_sample_mean": -2.81235933303833,
+      "k1_sample_min": -3.547999858856201,
+      "k1_sample_neg_pct": 58.75,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.03039727509021759,
+      "mask3_pct": 1.3726557493209839,
+      "neg_student_nll_loss": 0.03148110955953598,
+      "sampled_diff_pct": 1.3726557493209839,
+      "step": 160
+    },
+    {
+      "adv1_neg_pct": 0.4895250201225281,
+      "adv2_active_pct": 1.1766502857208252,
+      "ce_loss": 0.02162190154194832,
+      "epoch": 0.5743917032309533,
+      "grad_norm": 2.528654098510742,
+      "hpd_loss": 0.011955421417951584,
+      "k1_gt_raw_max": 0.20187309384346008,
+      "k1_gt_raw_mean": 0.008551979437470436,
+      "k1_gt_raw_min": -0.0070910281501710415,
+      "k1_gt_raw_neg_pct": 0.4895250201225281,
+      "k1_sample_max": -1.979987621307373,
+      "k1_sample_mean": -2.5711493492126465,
+      "k1_sample_min": -3.2561821937561035,
+      "k1_sample_neg_pct": 56.09375,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.011955419182777404,
+      "mask3_pct": 1.1766502857208252,
+      "neg_student_nll_loss": 0.02162190154194832,
+      "sampled_diff_pct": 1.185581922531128,
+      "step": 180
+    },
+    {
+      "adv1_neg_pct": 0.5023188591003418,
+      "adv2_active_pct": 1.5687706470489502,
+      "ce_loss": 0.027880553156137466,
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.9650227427482605,
+      "hpd_loss": 0.010340285487473011,
+      "k1_gt_raw_max": 0.2007327377796173,
+      "k1_gt_raw_mean": 0.009667925536632538,
+      "k1_gt_raw_min": -0.01589183136820793,
+      "k1_gt_raw_neg_pct": 0.5023188591003418,
+      "k1_sample_max": -1.5354162454605103,
+      "k1_sample_mean": -2.1268630027770996,
+      "k1_sample_min": -2.771559238433838,
+      "k1_sample_neg_pct": 56.40625,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.010340283811092376,
+      "mask3_pct": 1.5687706470489502,
+      "neg_student_nll_loss": 0.027880553156137466,
+      "sampled_diff_pct": 1.5725816488265991,
+      "step": 200
+    },
+    {
+      "adv1_neg_pct": 0.48300591111183167,
+      "adv2_active_pct": 1.1448569297790527,
+      "ce_loss": 0.023295864462852478,
+      "epoch": 0.702034303948943,
+      "grad_norm": 2.59177303314209,
+      "hpd_loss": 0.019972147420048714,
+      "k1_gt_raw_max": 0.18090441823005676,
+      "k1_gt_raw_mean": 0.0075406162068247795,
+      "k1_gt_raw_min": -0.014693280681967735,
+      "k1_gt_raw_neg_pct": 0.48300591111183167,
+      "k1_sample_max": -1.6416490077972412,
+      "k1_sample_mean": -2.2296323776245117,
+      "k1_sample_min": -2.930915355682373,
+      "k1_sample_neg_pct": 48.515625,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01997213959693909,
+      "mask3_pct": 1.1448569297790527,
+      "neg_student_nll_loss": 0.023295864462852478,
+      "sampled_diff_pct": 1.154510736465454,
+      "step": 220
+    },
+    {
+      "adv1_neg_pct": 0.765082597732544,
+      "adv2_active_pct": 0.9244414567947388,
+      "ce_loss": 0.02368738129734993,
+      "epoch": 0.7658556043079378,
+      "grad_norm": 1.1686080694198608,
+      "hpd_loss": 0.02641882374882698,
+      "k1_gt_raw_max": 0.1654866635799408,
+      "k1_gt_raw_mean": 0.005874675698578358,
+      "k1_gt_raw_min": -0.009244760498404503,
+      "k1_gt_raw_neg_pct": 0.765082597732544,
+      "k1_sample_max": -1.3134702444076538,
+      "k1_sample_mean": -1.9339661598205566,
+      "k1_sample_min": -2.6181559562683105,
+      "k1_sample_neg_pct": 45.3125,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.026418817043304444,
+      "mask3_pct": 0.9244414567947388,
+      "neg_student_nll_loss": 0.02368738129734993,
+      "sampled_diff_pct": 0.9244414567947388,
+      "step": 240
+    },
+    {
+      "adv1_neg_pct": 0.8438892364501953,
+      "adv2_active_pct": 0.929736852645874,
+      "ce_loss": 0.020094729959964752,
+      "epoch": 0.8296769046669326,
+      "grad_norm": 1.2779905796051025,
+      "hpd_loss": 0.015618223696947098,
+      "k1_gt_raw_max": 0.16853322088718414,
+      "k1_gt_raw_mean": 0.006190822925418615,
+      "k1_gt_raw_min": -0.009831264615058899,
+      "k1_gt_raw_neg_pct": 0.8438892364501953,
+      "k1_sample_max": -1.3172353506088257,
+      "k1_sample_mean": -1.7796552181243896,
+      "k1_sample_min": -2.261643409729004,
+      "k1_sample_neg_pct": 44.95738983154297,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.015618222951889037,
+      "mask3_pct": 0.9239488840103149,
+      "neg_student_nll_loss": 0.020094729959964752,
+      "sampled_diff_pct": 0.9378187656402588,
+      "step": 260
+    },
+    {
+      "adv1_neg_pct": 0.9945487976074219,
+      "adv2_active_pct": 0.7392160892486572,
+      "ce_loss": 0.015887977555394173,
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.26237931847572327,
+      "hpd_loss": 0.015452454797923565,
+      "k1_gt_raw_max": 0.14525339007377625,
+      "k1_gt_raw_mean": 0.0046372562646865845,
+      "k1_gt_raw_min": -0.003686224576085806,
+      "k1_gt_raw_neg_pct": 0.9945487976074219,
+      "k1_sample_max": -1.206074595451355,
+      "k1_sample_mean": -1.4332499504089355,
+      "k1_sample_min": -1.671927571296692,
+      "k1_sample_neg_pct": 35.625,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.015452450513839722,
+      "mask3_pct": 0.7392160892486572,
+      "neg_student_nll_loss": 0.015887977555394173,
+      "sampled_diff_pct": 0.7392160892486572,
+      "step": 280
+    },
+    {
+      "adv1_neg_pct": 0.9700182676315308,
+      "adv2_active_pct": 0.6018879413604736,
+      "ce_loss": 0.014513559639453888,
+      "epoch": 0.9573195053849222,
+      "grad_norm": 2.0338079929351807,
+      "hpd_loss": 0.013913260772824287,
+      "k1_gt_raw_max": 0.13846856355667114,
+      "k1_gt_raw_mean": 0.004541991278529167,
+      "k1_gt_raw_min": -0.002533096820116043,
+      "k1_gt_raw_neg_pct": 0.9700182676315308,
+      "k1_sample_max": -1.2417628765106201,
+      "k1_sample_mean": -1.4318351745605469,
+      "k1_sample_min": -1.6163195371627808,
+      "k1_sample_neg_pct": 32.864585876464844,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013913258910179138,
+      "mask3_pct": 0.6018879413604736,
+      "neg_student_nll_loss": 0.014513559639453888,
+      "sampled_diff_pct": 0.607985258102417,
+      "step": 300
+    },
+    {
+      "adv1_neg_pct": 0.8508870601654053,
+      "adv2_active_pct": 0.859713077545166,
+      "ce_loss": 0.019685909152030945,
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 86.6822,
+      "eval_generator_rouge1": 99.4006,
+      "eval_generator_rouge2": 99.0209,
+      "eval_generator_rougeL": 98.951,
+      "eval_loss": 0.010082065127789974,
+      "eval_runtime": 595.0313,
+      "eval_samples_per_second": 5.738,
+      "eval_selector_accuracy": 93.9836,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.18,
+      "hpd_loss": 0.02267773076891899,
+      "k1_gt_raw_max": 0.14172008633613586,
+      "k1_gt_raw_mean": 0.0054550133645534515,
+      "k1_gt_raw_min": -0.005414052866399288,
+      "k1_gt_raw_neg_pct": 0.8508870601654053,
+      "k1_sample_max": -1.4249026775360107,
+      "k1_sample_mean": -1.6359399557113647,
+      "k1_sample_min": -1.8393499851226807,
+      "k1_sample_neg_pct": 36.4485969543457,
+      "mask3_pct": 0.859713077545166,
+      "neg_student_nll_loss": 0.019685909152030945,
+      "sampled_diff_pct": 0.859713077545166,
+      "step": 314
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.172842003562496e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/hpd/checkpoint-314/training_args.bin b/qwen3/hpd/checkpoint-314/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1caa0e72cc5edd590e105073725cf4d832684e44
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f04fac21a72f04221af020c3bb88a0dc0ccf53bb7a0c0da21d1ab444937cb9d
+size 7185
diff --git a/qwen3/hpd/checkpoint-314/zero_to_fp32.py b/qwen3/hpd/checkpoint-314/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/hpd/checkpoint-314/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/hpd/checkpoint-628/README.md b/qwen3/hpd/checkpoint-628/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-628/adapter_config.json b/qwen3/hpd/checkpoint-628/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0f6594b49d741b9fe4eb4a1a4df0b797e15bcd
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-628/adapter_model.safetensors b/qwen3/hpd/checkpoint-628/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ec022f005b11f6a691197506aff4ac7be2936f69
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c94ae8896db0c35d2d477832d7bed178aadafff35918fd9c850196cce340080f
+size 40422560
diff --git a/qwen3/hpd/checkpoint-628/chat_template.jinja b/qwen3/hpd/checkpoint-628/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b3d0e371e3ca7dc29b8f79a12ddda5219635af6c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55657fc7721a3139830f872e8798110fc2e520f4860904ac08dc7fffb3e4372e
+size 121135557
diff --git a/qwen3/hpd/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c22d0a3f2bb1ac082c05c720fdc603ef2b7abd2
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e36e74e00a3d737e1c6975b2a1cb20dd45abb64a45a61f6b5175f397a086ade
+size 121135749
diff --git a/qwen3/hpd/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/hpd/checkpoint-628/global_step628/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5b4359db0deb5a131c00e82ca95710db3434cb6e
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/global_step628/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe899505b2ec17ee7fdd839b8f19b8f77f84d9a8bcc301d7656d1033a5832dbb
+size 351841765
diff --git a/qwen3/hpd/checkpoint-628/latest b/qwen3/hpd/checkpoint-628/latest
new file mode 100644
index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/latest
@@ -0,0 +1 @@
+global_step628
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-628/rng_state_0.pth b/qwen3/hpd/checkpoint-628/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5d7139c3d56e1594f73205c7b02e969eaff3a85a
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e6cf7abb420f243114a2275dc55226e0b0a9c4b1cf2df462cec83d08de00b81
+size 14917
diff --git a/qwen3/hpd/checkpoint-628/rng_state_1.pth b/qwen3/hpd/checkpoint-628/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d3ce716fbd0fdcd6a181622ea84344a33eb21983
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:222712eaa01a6df56dd8a18e193119109881d35e2a118891752635e28199508b
+size 14917
diff --git a/qwen3/hpd/checkpoint-628/scheduler.pt b/qwen3/hpd/checkpoint-628/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a
+size 1465
diff --git a/qwen3/hpd/checkpoint-628/tokenizer.json b/qwen3/hpd/checkpoint-628/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/hpd/checkpoint-628/tokenizer_config.json b/qwen3/hpd/checkpoint-628/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/hpd/checkpoint-628/trainer_state.json b/qwen3/hpd/checkpoint-628/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9878bfb29a981fc88968334cc17d446d16252c41
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/trainer_state.json
@@ -0,0 +1,776 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 628,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "adv1_neg_pct": 0.2358350306749344,
+      "adv2_active_pct": 17.716575622558594,
+      "ce_loss": 0.6496895551681519,
+      "epoch": 0.06382130035899482,
+      "grad_norm": 4.9955339431762695,
+      "hpd_loss": 0.7524586915969849,
+      "k1_gt_raw_max": 0.3395599126815796,
+      "k1_gt_raw_mean": 0.08363302797079086,
+      "k1_gt_raw_min": -0.006161125842481852,
+      "k1_gt_raw_neg_pct": 0.2358350306749344,
+      "k1_sample_max": -1.2287571430206299,
+      "k1_sample_mean": -5.443553924560547,
+      "k1_sample_min": -17.516345977783203,
+      "k1_sample_neg_pct": 97.79017639160156,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.7524587631225585,
+      "mask3_pct": 17.716575622558594,
+      "neg_student_nll_loss": 0.6496895551681519,
+      "sampled_diff_pct": 17.718591690063477,
+      "step": 20
+    },
+    {
+      "adv1_neg_pct": 0.24569129943847656,
+      "adv2_active_pct": 6.35067081451416,
+      "ce_loss": 0.12904959917068481,
+      "epoch": 0.12764260071798963,
+      "grad_norm": 3.5179519653320312,
+      "hpd_loss": 0.07163389772176743,
+      "k1_gt_raw_max": 0.3387124538421631,
+      "k1_gt_raw_mean": 0.03725019469857216,
+      "k1_gt_raw_min": -0.009099474176764488,
+      "k1_gt_raw_neg_pct": 0.24569129943847656,
+      "k1_sample_max": -1.1939605474472046,
+      "k1_sample_mean": -4.561788082122803,
+      "k1_sample_min": -10.153608322143555,
+      "k1_sample_neg_pct": 96.875,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.07163387537002563,
+      "mask3_pct": 6.35067081451416,
+      "neg_student_nll_loss": 0.12904959917068481,
+      "sampled_diff_pct": 6.35067081451416,
+      "step": 40
+    },
+    {
+      "adv1_neg_pct": 0.18458101153373718,
+      "adv2_active_pct": 5.008382320404053,
+      "ce_loss": 0.10459579527378082,
+      "epoch": 0.19146390107698444,
+      "grad_norm": 2.890872001647949,
+      "hpd_loss": 0.09348365664482117,
+      "k1_gt_raw_max": 0.3109760880470276,
+      "k1_gt_raw_mean": 0.026082728058099747,
+      "k1_gt_raw_min": -0.01705370843410492,
+      "k1_gt_raw_neg_pct": 0.18458101153373718,
+      "k1_sample_max": -2.0660922527313232,
+      "k1_sample_mean": -4.749420166015625,
+      "k1_sample_min": -8.562150955200195,
+      "k1_sample_neg_pct": 89.6875,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.09348366260528565,
+      "mask3_pct": 5.008382320404053,
+      "neg_student_nll_loss": 0.10459579527378082,
+      "sampled_diff_pct": 5.008382320404053,
+      "step": 60
+    },
+    {
+      "adv1_neg_pct": 0.3009452819824219,
+      "adv2_active_pct": 3.9314260482788086,
+      "ce_loss": 0.08063317835330963,
+      "epoch": 0.25528520143597927,
+      "grad_norm": 2.033250570297241,
+      "hpd_loss": 0.08107596635818481,
+      "k1_gt_raw_max": 0.2881737947463989,
+      "k1_gt_raw_mean": 0.019415946677327156,
+      "k1_gt_raw_min": -0.008919830434024334,
+      "k1_gt_raw_neg_pct": 0.3009452819824219,
+      "k1_sample_max": -1.6159262657165527,
+      "k1_sample_mean": -3.7506155967712402,
+      "k1_sample_min": -6.360409259796143,
+      "k1_sample_neg_pct": 86.39583587646484,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.0810759723186493,
+      "mask3_pct": 3.9314260482788086,
+      "neg_student_nll_loss": 0.08063317835330963,
+      "sampled_diff_pct": 3.9378738403320312,
+      "step": 80
+    },
+    {
+      "adv1_neg_pct": 0.22911369800567627,
+      "adv2_active_pct": 2.694115161895752,
+      "ce_loss": 0.052447859197854996,
+      "epoch": 0.3191065017949741,
+      "grad_norm": 1.751448392868042,
+      "hpd_loss": 0.027988094836473465,
+      "k1_gt_raw_max": 0.3031776547431946,
+      "k1_gt_raw_mean": 0.01864592172205448,
+      "k1_gt_raw_min": -0.007533406838774681,
+      "k1_gt_raw_neg_pct": 0.22911369800567627,
+      "k1_sample_max": -1.57374906539917,
+      "k1_sample_mean": -3.1450629234313965,
+      "k1_sample_min": -5.145340919494629,
+      "k1_sample_neg_pct": 84.70052337646484,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.027988094091415405,
+      "mask3_pct": 2.6920177936553955,
+      "neg_student_nll_loss": 0.052447859197854996,
+      "sampled_diff_pct": 2.7128100395202637,
+      "step": 100
+    },
+    {
+      "adv1_neg_pct": 0.1952832043170929,
+      "adv2_active_pct": 3.306915044784546,
+      "ce_loss": 0.04980571195483208,
+      "epoch": 0.3829278021539689,
+      "grad_norm": 3.930035352706909,
+      "hpd_loss": 0.015151199884712696,
+      "k1_gt_raw_max": 0.28564560413360596,
+      "k1_gt_raw_mean": 0.01884515956044197,
+      "k1_gt_raw_min": -0.001684204675257206,
+      "k1_gt_raw_neg_pct": 0.1952832043170929,
+      "k1_sample_max": -1.9181127548217773,
+      "k1_sample_mean": -3.2553763389587402,
+      "k1_sample_min": -4.8449602127075195,
+      "k1_sample_neg_pct": 75.20833587646484,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.015151196718215942,
+      "mask3_pct": 3.306915044784546,
+      "neg_student_nll_loss": 0.04980571195483208,
+      "sampled_diff_pct": 3.3092129230499268,
+      "step": 120
+    },
+    {
+      "adv1_neg_pct": 0.3264433741569519,
+      "adv2_active_pct": 2.25433087348938,
+      "ce_loss": 0.04018855467438698,
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.6639359593391418,
+      "hpd_loss": 0.02619136869907379,
+      "k1_gt_raw_max": 0.24626323580741882,
+      "k1_gt_raw_mean": 0.015010751783847809,
+      "k1_gt_raw_min": -0.004701526835560799,
+      "k1_gt_raw_neg_pct": 0.3264433741569519,
+      "k1_sample_max": -1.902454137802124,
+      "k1_sample_mean": -2.805896282196045,
+      "k1_sample_min": -3.829012393951416,
+      "k1_sample_neg_pct": 71.25,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.026191365718841553,
+      "mask3_pct": 2.25433087348938,
+      "neg_student_nll_loss": 0.04018855467438698,
+      "sampled_diff_pct": 2.2565953731536865,
+      "step": 140
+    },
+    {
+      "adv1_neg_pct": 0.3137112259864807,
+      "adv2_active_pct": 1.3726557493209839,
+      "ce_loss": 0.03148110955953598,
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.548140525817871,
+      "hpd_loss": 0.03039727918803692,
+      "k1_gt_raw_max": 0.20344290137290955,
+      "k1_gt_raw_mean": 0.009032711386680603,
+      "k1_gt_raw_min": -0.0009857998229563236,
+      "k1_gt_raw_neg_pct": 0.3137112259864807,
+      "k1_sample_max": -2.153831958770752,
+      "k1_sample_mean": -2.81235933303833,
+      "k1_sample_min": -3.547999858856201,
+      "k1_sample_neg_pct": 58.75,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.03039727509021759,
+      "mask3_pct": 1.3726557493209839,
+      "neg_student_nll_loss": 0.03148110955953598,
+      "sampled_diff_pct": 1.3726557493209839,
+      "step": 160
+    },
+    {
+      "adv1_neg_pct": 0.4895250201225281,
+      "adv2_active_pct": 1.1766502857208252,
+      "ce_loss": 0.02162190154194832,
+      "epoch": 0.5743917032309533,
+      "grad_norm": 2.528654098510742,
+      "hpd_loss": 0.011955421417951584,
+      "k1_gt_raw_max": 0.20187309384346008,
+      "k1_gt_raw_mean": 0.008551979437470436,
+      "k1_gt_raw_min": -0.0070910281501710415,
+      "k1_gt_raw_neg_pct": 0.4895250201225281,
+      "k1_sample_max": -1.979987621307373,
+      "k1_sample_mean": -2.5711493492126465,
+      "k1_sample_min": -3.2561821937561035,
+      "k1_sample_neg_pct": 56.09375,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.011955419182777404,
+      "mask3_pct": 1.1766502857208252,
+      "neg_student_nll_loss": 0.02162190154194832,
+      "sampled_diff_pct": 1.185581922531128,
+      "step": 180
+    },
+    {
+      "adv1_neg_pct": 0.5023188591003418,
+      "adv2_active_pct": 1.5687706470489502,
+      "ce_loss": 0.027880553156137466,
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.9650227427482605,
+      "hpd_loss": 0.010340285487473011,
+      "k1_gt_raw_max": 0.2007327377796173,
+      "k1_gt_raw_mean": 0.009667925536632538,
+      "k1_gt_raw_min": -0.01589183136820793,
+      "k1_gt_raw_neg_pct": 0.5023188591003418,
+      "k1_sample_max": -1.5354162454605103,
+      "k1_sample_mean": -2.1268630027770996,
+      "k1_sample_min": -2.771559238433838,
+      "k1_sample_neg_pct": 56.40625,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.010340283811092376,
+      "mask3_pct": 1.5687706470489502,
+      "neg_student_nll_loss": 0.027880553156137466,
+      "sampled_diff_pct": 1.5725816488265991,
+      "step": 200
+    },
+    {
+      "adv1_neg_pct": 0.48300591111183167,
+      "adv2_active_pct": 1.1448569297790527,
+      "ce_loss": 0.023295864462852478,
+      "epoch": 0.702034303948943,
+      "grad_norm": 2.59177303314209,
+      "hpd_loss": 0.019972147420048714,
+      "k1_gt_raw_max": 0.18090441823005676,
+      "k1_gt_raw_mean": 0.0075406162068247795,
+      "k1_gt_raw_min": -0.014693280681967735,
+      "k1_gt_raw_neg_pct": 0.48300591111183167,
+      "k1_sample_max": -1.6416490077972412,
+      "k1_sample_mean": -2.2296323776245117,
+      "k1_sample_min": -2.930915355682373,
+      "k1_sample_neg_pct": 48.515625,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01997213959693909,
+      "mask3_pct": 1.1448569297790527,
+      "neg_student_nll_loss": 0.023295864462852478,
+      "sampled_diff_pct": 1.154510736465454,
+      "step": 220
+    },
+    {
+      "adv1_neg_pct": 0.765082597732544,
+      "adv2_active_pct": 0.9244414567947388,
+      "ce_loss": 0.02368738129734993,
+      "epoch": 0.7658556043079378,
+      "grad_norm": 1.1686080694198608,
+      "hpd_loss": 0.02641882374882698,
+      "k1_gt_raw_max": 0.1654866635799408,
+      "k1_gt_raw_mean": 0.005874675698578358,
+      "k1_gt_raw_min": -0.009244760498404503,
+      "k1_gt_raw_neg_pct": 0.765082597732544,
+      "k1_sample_max": -1.3134702444076538,
+      "k1_sample_mean": -1.9339661598205566,
+      "k1_sample_min": -2.6181559562683105,
+      "k1_sample_neg_pct": 45.3125,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.026418817043304444,
+      "mask3_pct": 0.9244414567947388,
+      "neg_student_nll_loss": 0.02368738129734993,
+      "sampled_diff_pct": 0.9244414567947388,
+      "step": 240
+    },
+    {
+      "adv1_neg_pct": 0.8438892364501953,
+      "adv2_active_pct": 0.929736852645874,
+      "ce_loss": 0.020094729959964752,
+      "epoch": 0.8296769046669326,
+      "grad_norm": 1.2779905796051025,
+      "hpd_loss": 0.015618223696947098,
+      "k1_gt_raw_max": 0.16853322088718414,
+      "k1_gt_raw_mean": 0.006190822925418615,
+      "k1_gt_raw_min": -0.009831264615058899,
+      "k1_gt_raw_neg_pct": 0.8438892364501953,
+      "k1_sample_max": -1.3172353506088257,
+      "k1_sample_mean": -1.7796552181243896,
+      "k1_sample_min": -2.261643409729004,
+      "k1_sample_neg_pct": 44.95738983154297,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.015618222951889037,
+      "mask3_pct": 0.9239488840103149,
+      "neg_student_nll_loss": 0.020094729959964752,
+      "sampled_diff_pct": 0.9378187656402588,
+      "step": 260
+    },
+    {
+      "adv1_neg_pct": 0.9945487976074219,
+      "adv2_active_pct": 0.7392160892486572,
+      "ce_loss": 0.015887977555394173,
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.26237931847572327,
+      "hpd_loss": 0.015452454797923565,
+      "k1_gt_raw_max": 0.14525339007377625,
+      "k1_gt_raw_mean": 0.0046372562646865845,
+      "k1_gt_raw_min": -0.003686224576085806,
+      "k1_gt_raw_neg_pct": 0.9945487976074219,
+      "k1_sample_max": -1.206074595451355,
+      "k1_sample_mean": -1.4332499504089355,
+      "k1_sample_min": -1.671927571296692,
+      "k1_sample_neg_pct": 35.625,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.015452450513839722,
+      "mask3_pct": 0.7392160892486572,
+      "neg_student_nll_loss": 0.015887977555394173,
+      "sampled_diff_pct": 0.7392160892486572,
+      "step": 280
+    },
+    {
+      "adv1_neg_pct": 0.9700182676315308,
+      "adv2_active_pct": 0.6018879413604736,
+      "ce_loss": 0.014513559639453888,
+      "epoch": 0.9573195053849222,
+      "grad_norm": 2.0338079929351807,
+      "hpd_loss": 0.013913260772824287,
+      "k1_gt_raw_max": 0.13846856355667114,
+      "k1_gt_raw_mean": 0.004541991278529167,
+      "k1_gt_raw_min": -0.002533096820116043,
+      "k1_gt_raw_neg_pct": 0.9700182676315308,
+      "k1_sample_max": -1.2417628765106201,
+      "k1_sample_mean": -1.4318351745605469,
+      "k1_sample_min": -1.6163195371627808,
+      "k1_sample_neg_pct": 32.864585876464844,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013913258910179138,
+      "mask3_pct": 0.6018879413604736,
+      "neg_student_nll_loss": 0.014513559639453888,
+      "sampled_diff_pct": 0.607985258102417,
+      "step": 300
+    },
+    {
+      "adv1_neg_pct": 0.8508870601654053,
+      "adv2_active_pct": 0.859713077545166,
+      "ce_loss": 0.019685909152030945,
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 86.6822,
+      "eval_generator_rouge1": 99.4006,
+      "eval_generator_rouge2": 99.0209,
+      "eval_generator_rougeL": 98.951,
+      "eval_loss": 0.010082065127789974,
+      "eval_runtime": 595.0313,
+      "eval_samples_per_second": 5.738,
+      "eval_selector_accuracy": 93.9836,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.18,
+      "hpd_loss": 0.02267773076891899,
+      "k1_gt_raw_max": 0.14172008633613586,
+      "k1_gt_raw_mean": 0.0054550133645534515,
+      "k1_gt_raw_min": -0.005414052866399288,
+      "k1_gt_raw_neg_pct": 0.8508870601654053,
+      "k1_sample_max": -1.4249026775360107,
+      "k1_sample_mean": -1.6359399557113647,
+      "k1_sample_min": -1.8393499851226807,
+      "k1_sample_neg_pct": 36.4485969543457,
+      "mask3_pct": 0.859713077545166,
+      "neg_student_nll_loss": 0.019685909152030945,
+      "sampled_diff_pct": 0.859713077545166,
+      "step": 314
+    },
+    {
+      "adv1_neg_pct": 0.6673219203948975,
+      "adv2_active_pct": 1.2173185348510742,
+      "ce_loss": 0.011882005259394646,
+      "epoch": 1.0191463901076985,
+      "grad_norm": 4.386560916900635,
+      "hpd_loss": -0.0019697409588843584,
+      "k1_gt_raw_max": 0.14985336363315582,
+      "k1_gt_raw_mean": 0.006570600904524326,
+      "k1_gt_raw_min": -0.0011412458261474967,
+      "k1_gt_raw_neg_pct": 0.6673219203948975,
+      "k1_sample_max": -0.5733932256698608,
+      "k1_sample_mean": -0.7650949358940125,
+      "k1_sample_min": -0.9706472158432007,
+      "k1_sample_neg_pct": 36.458335876464844,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014574740827083588,
+      "mask3_pct": 1.2173185348510742,
+      "neg_student_nll_loss": 0.011882005259394646,
+      "sampled_diff_pct": 1.2173185348510742,
+      "step": 320
+    },
+    {
+      "adv1_neg_pct": 1.1466679573059082,
+      "adv2_active_pct": 0.4476942718029022,
+      "ce_loss": 0.009723825380206108,
+      "epoch": 1.0829676904666932,
+      "grad_norm": 1.2898244857788086,
+      "hpd_loss": 0.009684150107204914,
+      "k1_gt_raw_max": 0.10980867594480515,
+      "k1_gt_raw_mean": 0.0032046493142843246,
+      "k1_gt_raw_min": -0.009925473481416702,
+      "k1_gt_raw_neg_pct": 1.1466679573059082,
+      "k1_sample_max": -0.76816326379776,
+      "k1_sample_mean": -0.8626709580421448,
+      "k1_sample_min": -0.9573334455490112,
+      "k1_sample_neg_pct": 28.95833396911621,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.0096841499209404,
+      "mask3_pct": 0.4476942718029022,
+      "neg_student_nll_loss": 0.009723825380206108,
+      "sampled_diff_pct": 0.4520599842071533,
+      "step": 340
+    },
+    {
+      "adv1_neg_pct": 1.657383918762207,
+      "adv2_active_pct": 0.7416136860847473,
+      "ce_loss": 0.01363883912563324,
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.8189982175827026,
+      "hpd_loss": 0.012504098936915398,
+      "k1_gt_raw_max": 0.11255955696105957,
+      "k1_gt_raw_mean": 0.0030380175448954105,
+      "k1_gt_raw_min": -0.01135759986937046,
+      "k1_gt_raw_neg_pct": 1.657383918762207,
+      "k1_sample_max": -1.1351618766784668,
+      "k1_sample_mean": -1.3799619674682617,
+      "k1_sample_min": -1.6486986875534058,
+      "k1_sample_neg_pct": 30.0,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.012504099309444428,
+      "mask3_pct": 0.7416136860847473,
+      "neg_student_nll_loss": 0.01363883912563324,
+      "sampled_diff_pct": 0.7438143491744995,
+      "step": 360
+    },
+    {
+      "adv1_neg_pct": 1.3510011434555054,
+      "adv2_active_pct": 0.3445201814174652,
+      "ce_loss": 0.007902171462774277,
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.5892553329467773,
+      "hpd_loss": 0.008744428865611553,
+      "k1_gt_raw_max": 0.09419713914394379,
+      "k1_gt_raw_mean": 0.0022562379017472267,
+      "k1_gt_raw_min": -0.006403174251317978,
+      "k1_gt_raw_neg_pct": 1.3510011434555054,
+      "k1_sample_max": -0.583869457244873,
+      "k1_sample_mean": -0.7341683506965637,
+      "k1_sample_min": -0.8994753360748291,
+      "k1_sample_neg_pct": 23.75,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.008744429051876067,
+      "mask3_pct": 0.3445201814174652,
+      "neg_student_nll_loss": 0.007902171462774277,
+      "sampled_diff_pct": 0.35682976245880127,
+      "step": 380
+    },
+    {
+      "adv1_neg_pct": 1.4141113758087158,
+      "adv2_active_pct": 0.3374101221561432,
+      "ce_loss": 0.008193380199372768,
+      "epoch": 1.2744315915436777,
+      "grad_norm": 1.2929012775421143,
+      "hpd_loss": 0.006689602509140968,
+      "k1_gt_raw_max": 0.10671139508485794,
+      "k1_gt_raw_mean": 0.002954404801130295,
+      "k1_gt_raw_min": -0.006222248077392578,
+      "k1_gt_raw_neg_pct": 1.4141114950180054,
+      "k1_sample_max": -0.6661227941513062,
+      "k1_sample_mean": -0.7585095763206482,
+      "k1_sample_min": -0.8607568740844727,
+      "k1_sample_neg_pct": 25.760417938232422,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.0066896028816699985,
+      "mask3_pct": 0.3344895839691162,
+      "neg_student_nll_loss": 0.008193380199372768,
+      "sampled_diff_pct": 0.3449546694755554,
+      "step": 400
+    },
+    {
+      "adv1_neg_pct": 2.1220993995666504,
+      "adv2_active_pct": 0.4313049912452698,
+      "ce_loss": 0.011112527921795845,
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.8164143562316895,
+      "hpd_loss": 0.010808300226926804,
+      "k1_gt_raw_max": 0.10569541901350021,
+      "k1_gt_raw_mean": 0.0035689054057002068,
+      "k1_gt_raw_min": -0.004470471758395433,
+      "k1_gt_raw_neg_pct": 2.1220993995666504,
+      "k1_sample_max": -0.6986867189407349,
+      "k1_sample_mean": -0.7543543577194214,
+      "k1_sample_min": -0.8270018100738525,
+      "k1_sample_neg_pct": 25.279605865478516,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.010808297991752624,
+      "mask3_pct": 0.4313049912452698,
+      "neg_student_nll_loss": 0.011112527921795845,
+      "sampled_diff_pct": 0.4342671036720276,
+      "step": 420
+    },
+    {
+      "adv1_neg_pct": 1.9999275207519531,
+      "adv2_active_pct": 0.2967550456523895,
+      "ce_loss": 0.007173327729105949,
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.959374189376831,
+      "hpd_loss": 0.005844146944582462,
+      "k1_gt_raw_max": 0.08694229274988174,
+      "k1_gt_raw_mean": 0.002139701507985592,
+      "k1_gt_raw_min": -0.005928537808358669,
+      "k1_gt_raw_neg_pct": 1.9999275207519531,
+      "k1_sample_max": -0.8034830093383789,
+      "k1_sample_mean": -0.8980427980422974,
+      "k1_sample_min": -0.9926026463508606,
+      "k1_sample_neg_pct": 20.78125,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.005844147875905037,
+      "mask3_pct": 0.2967550456523895,
+      "neg_student_nll_loss": 0.007173327729105949,
+      "sampled_diff_pct": 0.30670204758644104,
+      "step": 440
+    },
+    {
+      "adv1_neg_pct": 1.9445667266845703,
+      "adv2_active_pct": 0.28356558084487915,
+      "ce_loss": 0.007787229493260384,
+      "epoch": 1.4658954926206622,
+      "grad_norm": 1.3586090803146362,
+      "hpd_loss": 0.009073459543287754,
+      "k1_gt_raw_max": 0.07425311952829361,
+      "k1_gt_raw_mean": 0.0016224482096731663,
+      "k1_gt_raw_min": -0.0022226697765290737,
+      "k1_gt_raw_neg_pct": 1.9445667266845703,
+      "k1_sample_max": -0.6454492807388306,
+      "k1_sample_mean": -0.686149001121521,
+      "k1_sample_min": -0.7300035953521729,
+      "k1_sample_neg_pct": 17.1875,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.009073460102081298,
+      "mask3_pct": 0.27911925315856934,
+      "neg_student_nll_loss": 0.007787229493260384,
+      "sampled_diff_pct": 0.28356558084487915,
+      "step": 460
+    },
+    {
+      "adv1_neg_pct": 1.8402546644210815,
+      "adv2_active_pct": 0.41872814297676086,
+      "ce_loss": 0.005175907164812088,
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.2023886740207672,
+      "hpd_loss": 0.0015706174308434129,
+      "k1_gt_raw_max": 0.07684825360774994,
+      "k1_gt_raw_mean": 0.0022208630107343197,
+      "k1_gt_raw_min": -0.008054896257817745,
+      "k1_gt_raw_neg_pct": 1.8402546644210815,
+      "k1_sample_max": -0.47196078300476074,
+      "k1_sample_mean": -0.4973880648612976,
+      "k1_sample_min": -0.5228154063224792,
+      "k1_sample_neg_pct": 17.5,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0015706172212958335,
+      "mask3_pct": 0.41872814297676086,
+      "neg_student_nll_loss": 0.005175907164812088,
+      "sampled_diff_pct": 0.4208254814147949,
+      "step": 480
+    },
+    {
+      "adv1_neg_pct": 2.5322799682617188,
+      "adv2_active_pct": 0.33805567026138306,
+      "ce_loss": 0.00585869699716568,
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.030877001583576202,
+      "hpd_loss": 0.003286010818555951,
+      "k1_gt_raw_max": 0.058261577039957047,
+      "k1_gt_raw_mean": 0.002064550295472145,
+      "k1_gt_raw_min": -0.0029995564837008715,
+      "k1_gt_raw_neg_pct": 2.5322799682617188,
+      "k1_sample_max": -0.3366956412792206,
+      "k1_sample_mean": -0.39731863141059875,
+      "k1_sample_min": -0.45794159173965454,
+      "k1_sample_neg_pct": 14.0625,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.0032860103994607924,
+      "mask3_pct": 0.33805567026138306,
+      "neg_student_nll_loss": 0.00585869699716568,
+      "sampled_diff_pct": 0.3419136703014374,
+      "step": 500
+    },
+    {
+      "adv1_neg_pct": 2.701911449432373,
+      "adv2_active_pct": 0.3643760681152344,
+      "ce_loss": 0.010007714852690697,
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.14292795956134796,
+      "hpd_loss": 0.012855440378189087,
+      "k1_gt_raw_max": 0.06845426559448242,
+      "k1_gt_raw_mean": 0.0017418819479644299,
+      "k1_gt_raw_min": -0.004030767362564802,
+      "k1_gt_raw_neg_pct": 2.701911449432373,
+      "k1_sample_max": -0.4802917242050171,
+      "k1_sample_mean": -0.5421380996704102,
+      "k1_sample_min": -0.6135058999061584,
+      "k1_sample_neg_pct": 17.03125,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.012855446338653565,
+      "mask3_pct": 0.3643760681152344,
+      "neg_student_nll_loss": 0.010007714852690697,
+      "sampled_diff_pct": 0.3710411787033081,
+      "step": 520
+    },
+    {
+      "adv1_neg_pct": 2.248600959777832,
+      "adv2_active_pct": 0.5108850598335266,
+      "ce_loss": 0.01003289595246315,
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.6681362390518188,
+      "hpd_loss": 0.009245548397302628,
+      "k1_gt_raw_max": 0.08858497440814972,
+      "k1_gt_raw_mean": 0.002853624988347292,
+      "k1_gt_raw_min": -0.004460392985492945,
+      "k1_gt_raw_neg_pct": 2.248600959777832,
+      "k1_sample_max": -0.7408795356750488,
+      "k1_sample_mean": -0.8169400691986084,
+      "k1_sample_min": -0.8906024694442749,
+      "k1_sample_neg_pct": 22.65625,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.009245550632476807,
+      "mask3_pct": 0.5108850598335266,
+      "neg_student_nll_loss": 0.01003289595246315,
+      "sampled_diff_pct": 0.5210334062576294,
+      "step": 540
+    },
+    {
+      "adv1_neg_pct": 2.4635496139526367,
+      "adv2_active_pct": 0.22115065157413483,
+      "ce_loss": 0.005004607141017914,
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.08243642747402191,
+      "hpd_loss": 0.0062531642615795135,
+      "k1_gt_raw_max": 0.0580272376537323,
+      "k1_gt_raw_mean": 0.0011763360816985369,
+      "k1_gt_raw_min": -0.005644598975777626,
+      "k1_gt_raw_neg_pct": 2.4635493755340576,
+      "k1_sample_max": -0.2404015064239502,
+      "k1_sample_mean": -0.2807300090789795,
+      "k1_sample_min": -0.32241004705429077,
+      "k1_sample_neg_pct": 14.921875,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.006253162026405334,
+      "mask3_pct": 0.22115065157413483,
+      "neg_student_nll_loss": 0.005004607141017914,
+      "sampled_diff_pct": 0.22733174264431,
+      "step": 560
+    },
+    {
+      "adv1_neg_pct": 3.3381173610687256,
+      "adv2_active_pct": 0.23258033394813538,
+      "ce_loss": 0.006839662324637175,
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.0869053527712822,
+      "hpd_loss": 0.00969111267477274,
+      "k1_gt_raw_max": 0.05798381567001343,
+      "k1_gt_raw_mean": 0.0016341875307261944,
+      "k1_gt_raw_min": -0.009443331509828568,
+      "k1_gt_raw_neg_pct": 3.3381173610687256,
+      "k1_sample_max": -0.29867684841156006,
+      "k1_sample_mean": -0.31859445571899414,
+      "k1_sample_min": -0.338512122631073,
+      "k1_sample_neg_pct": 10.625,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.009691111743450165,
+      "mask3_pct": 0.23258033394813538,
+      "neg_student_nll_loss": 0.006839662324637175,
+      "sampled_diff_pct": 0.24274389445781708,
+      "step": 580
+    },
+    {
+      "adv1_neg_pct": 4.040760040283203,
+      "adv2_active_pct": 0.20796719193458557,
+      "ce_loss": 0.002632163930684328,
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2512037456035614,
+      "hpd_loss": 0.0005975529784336686,
+      "k1_gt_raw_max": 0.049394719302654266,
+      "k1_gt_raw_mean": 0.0014062562258914113,
+      "k1_gt_raw_min": -0.002836518455296755,
+      "k1_gt_raw_neg_pct": 4.040760040283203,
+      "k1_sample_max": -0.23663434386253357,
+      "k1_sample_mean": -0.255886435508728,
+      "k1_sample_min": -0.2802615463733673,
+      "k1_sample_neg_pct": 10.208333969116211,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.0005975536536425352,
+      "mask3_pct": 0.20796719193458557,
+      "neg_student_nll_loss": 0.002632163930684328,
+      "sampled_diff_pct": 0.21867018938064575,
+      "step": 600
+    },
+    {
+      "adv1_neg_pct": 4.097904205322266,
+      "adv2_active_pct": 0.1848699152469635,
+      "ce_loss": 0.0032025063410401344,
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.338096022605896,
+      "hpd_loss": 0.002149906475096941,
+      "k1_gt_raw_max": 0.043635837733745575,
+      "k1_gt_raw_mean": 0.0007720005232840776,
+      "k1_gt_raw_min": -0.0036990540102124214,
+      "k1_gt_raw_neg_pct": 4.097904205322266,
+      "k1_sample_max": -0.2395995557308197,
+      "k1_sample_mean": -0.2737259864807129,
+      "k1_sample_min": -0.3078524172306061,
+      "k1_sample_neg_pct": 13.4375,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.0021499061957001688,
+      "mask3_pct": 0.1848699152469635,
+      "neg_student_nll_loss": 0.0032025063410401344,
+      "sampled_diff_pct": 0.18927133083343506,
+      "step": 620
+    },
+    {
+      "adv1_neg_pct": 3.390739917755127,
+      "adv2_active_pct": 0.10791555047035217,
+      "ce_loss": 0.001353576430119574,
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.1519,
+      "eval_generator_rouge1": 99.7488,
+      "eval_generator_rouge2": 99.6229,
+      "eval_generator_rougeL": 99.5782,
+      "eval_loss": 0.004672760609537363,
+      "eval_runtime": 596.1717,
+      "eval_samples_per_second": 5.727,
+      "eval_selector_accuracy": 98.014,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.179,
+      "hpd_loss": 0.0011793917510658503,
+      "k1_gt_raw_max": 0.03581507131457329,
+      "k1_gt_raw_mean": 0.0006900117732584476,
+      "k1_gt_raw_min": -0.002846459159627557,
+      "k1_gt_raw_neg_pct": 3.390739917755127,
+      "k1_sample_max": -0.06917751580476761,
+      "k1_sample_mean": -0.06961424648761749,
+      "k1_sample_min": -0.07005097717046738,
+      "k1_sample_neg_pct": 10.1694917678833,
+      "mask3_pct": 0.10791555047035217,
+      "neg_student_nll_loss": 0.001353576430119574,
+      "sampled_diff_pct": 0.10791555047035217,
+      "step": 628
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.346351770520781e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/hpd/checkpoint-628/training_args.bin b/qwen3/hpd/checkpoint-628/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1caa0e72cc5edd590e105073725cf4d832684e44
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f04fac21a72f04221af020c3bb88a0dc0ccf53bb7a0c0da21d1ab444937cb9d
+size 7185
diff --git a/qwen3/hpd/checkpoint-628/zero_to_fp32.py b/qwen3/hpd/checkpoint-628/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/hpd/checkpoint-628/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/hpd/checkpoint-942/README.md b/qwen3/hpd/checkpoint-942/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-942/adapter_config.json b/qwen3/hpd/checkpoint-942/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0f6594b49d741b9fe4eb4a1a4df0b797e15bcd
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/adapter_config.json
@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-942/adapter_model.safetensors b/qwen3/hpd/checkpoint-942/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..09a31e1dfbe0a4433be559d8520fe531b38312dc
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee956deed57c11e94d417657ae1051386592d8015ac8a3c6c9b88728bc91d141
+size 40422560
diff --git a/qwen3/hpd/checkpoint-942/chat_template.jinja b/qwen3/hpd/checkpoint-942/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d83770437083978835c2b30c24315de340482c05
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a23a677b0434645660b97cb016a5eb2060e48ee9b500b57d6143c044b3dddb7d
+size 121135557
diff --git a/qwen3/hpd/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/hpd/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0d4d9e2f804a1d96a34bd808959d93bceb055d3
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf1ae40ed50274ea1c792df1e8991e0655278fa5c1f0abaade7c4ea7a4bd813c
+size 121135749
diff --git a/qwen3/hpd/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/hpd/checkpoint-942/global_step942/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bc8eb741a1528f7d33af4b141dd86d9eb65c650
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/global_step942/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d439a13e25541fd0230469abea5ce0479417f7ef32ca2ce4f3752cb23b840bfc
+size 351841765
diff --git a/qwen3/hpd/checkpoint-942/latest b/qwen3/hpd/checkpoint-942/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/latest
@@ -0,0 +1 @@
+global_step942
\ No newline at end of file
diff --git a/qwen3/hpd/checkpoint-942/rng_state_0.pth b/qwen3/hpd/checkpoint-942/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e599d244daa70ff12d73336a36f4142563b3d471
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1592e7675a2e95220cb5f1ffbedb7297d9cd79d70a90633fd354ffdc3d07466
+size 14917
diff --git a/qwen3/hpd/checkpoint-942/rng_state_1.pth b/qwen3/hpd/checkpoint-942/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..75223b19af8c1e67b3c07df72b8602eed8caf47a
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4747341b89a7d0f1b6bf427da14490fa819468fc504ca6fde00e001475d76e37
+size 14917
diff --git a/qwen3/hpd/checkpoint-942/scheduler.pt b/qwen3/hpd/checkpoint-942/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3
+size 1465
diff --git a/qwen3/hpd/checkpoint-942/tokenizer.json b/qwen3/hpd/checkpoint-942/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/hpd/checkpoint-942/tokenizer_config.json b/qwen3/hpd/checkpoint-942/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/hpd/checkpoint-942/trainer_state.json b/qwen3/hpd/checkpoint-942/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..b58d843711bb14ea536dd9b8dc35111def6b1923
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/trainer_state.json
@@ -0,0 +1,1158 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 942,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "adv1_neg_pct": 0.2358350306749344,
+      "adv2_active_pct": 17.716575622558594,
+      "ce_loss": 0.6496895551681519,
+      "epoch": 0.06382130035899482,
+      "grad_norm": 4.9955339431762695,
+      "hpd_loss": 0.7524586915969849,
+      "k1_gt_raw_max": 0.3395599126815796,
+      "k1_gt_raw_mean": 0.08363302797079086,
+      "k1_gt_raw_min": -0.006161125842481852,
+      "k1_gt_raw_neg_pct": 0.2358350306749344,
+      "k1_sample_max": -1.2287571430206299,
+      "k1_sample_mean": -5.443553924560547,
+      "k1_sample_min": -17.516345977783203,
+      "k1_sample_neg_pct": 97.79017639160156,
+      "learning_rate": 9.996390385474569e-05,
+      "loss": 0.7524587631225585,
+      "mask3_pct": 17.716575622558594,
+      "neg_student_nll_loss": 0.6496895551681519,
+      "sampled_diff_pct": 17.718591690063477,
+      "step": 20
+    },
+    {
+      "adv1_neg_pct": 0.24569129943847656,
+      "adv2_active_pct": 6.35067081451416,
+      "ce_loss": 0.12904959917068481,
+      "epoch": 0.12764260071798963,
+      "grad_norm": 3.5179519653320312,
+      "hpd_loss": 0.07163389772176743,
+      "k1_gt_raw_max": 0.3387124538421631,
+      "k1_gt_raw_mean": 0.03725019469857216,
+      "k1_gt_raw_min": -0.009099474176764488,
+      "k1_gt_raw_neg_pct": 0.24569129943847656,
+      "k1_sample_max": -1.1939605474472046,
+      "k1_sample_mean": -4.561788082122803,
+      "k1_sample_min": -10.153608322143555,
+      "k1_sample_neg_pct": 96.875,
+      "learning_rate": 9.984797509888031e-05,
+      "loss": 0.07163387537002563,
+      "mask3_pct": 6.35067081451416,
+      "neg_student_nll_loss": 0.12904959917068481,
+      "sampled_diff_pct": 6.35067081451416,
+      "step": 40
+    },
+    {
+      "adv1_neg_pct": 0.18458101153373718,
+      "adv2_active_pct": 5.008382320404053,
+      "ce_loss": 0.10459579527378082,
+      "epoch": 0.19146390107698444,
+      "grad_norm": 2.890872001647949,
+      "hpd_loss": 0.09348365664482117,
+      "k1_gt_raw_max": 0.3109760880470276,
+      "k1_gt_raw_mean": 0.026082728058099747,
+      "k1_gt_raw_min": -0.01705370843410492,
+      "k1_gt_raw_neg_pct": 0.18458101153373718,
+      "k1_sample_max": -2.0660922527313232,
+      "k1_sample_mean": -4.749420166015625,
+      "k1_sample_min": -8.562150955200195,
+      "k1_sample_neg_pct": 89.6875,
+      "learning_rate": 9.965229938050925e-05,
+      "loss": 0.09348366260528565,
+      "mask3_pct": 5.008382320404053,
+      "neg_student_nll_loss": 0.10459579527378082,
+      "sampled_diff_pct": 5.008382320404053,
+      "step": 60
+    },
+    {
+      "adv1_neg_pct": 0.3009452819824219,
+      "adv2_active_pct": 3.9314260482788086,
+      "ce_loss": 0.08063317835330963,
+      "epoch": 0.25528520143597927,
+      "grad_norm": 2.033250570297241,
+      "hpd_loss": 0.08107596635818481,
+      "k1_gt_raw_max": 0.2881737947463989,
+      "k1_gt_raw_mean": 0.019415946677327156,
+      "k1_gt_raw_min": -0.008919830434024334,
+      "k1_gt_raw_neg_pct": 0.3009452819824219,
+      "k1_sample_max": -1.6159262657165527,
+      "k1_sample_mean": -3.7506155967712402,
+      "k1_sample_min": -6.360409259796143,
+      "k1_sample_neg_pct": 86.39583587646484,
+      "learning_rate": 9.937719005663432e-05,
+      "loss": 0.0810759723186493,
+      "mask3_pct": 3.9314260482788086,
+      "neg_student_nll_loss": 0.08063317835330963,
+      "sampled_diff_pct": 3.9378738403320312,
+      "step": 80
+    },
+    {
+      "adv1_neg_pct": 0.22911369800567627,
+      "adv2_active_pct": 2.694115161895752,
+      "ce_loss": 0.052447859197854996,
+      "epoch": 0.3191065017949741,
+      "grad_norm": 1.751448392868042,
+      "hpd_loss": 0.027988094836473465,
+      "k1_gt_raw_max": 0.3031776547431946,
+      "k1_gt_raw_mean": 0.01864592172205448,
+      "k1_gt_raw_min": -0.007533406838774681,
+      "k1_gt_raw_neg_pct": 0.22911369800567627,
+      "k1_sample_max": -1.57374906539917,
+      "k1_sample_mean": -3.1450629234313965,
+      "k1_sample_min": -5.145340919494629,
+      "k1_sample_neg_pct": 84.70052337646484,
+      "learning_rate": 9.902308769000724e-05,
+      "loss": 0.027988094091415405,
+      "mask3_pct": 2.6920177936553955,
+      "neg_student_nll_loss": 0.052447859197854996,
+      "sampled_diff_pct": 2.7128100395202637,
+      "step": 100
+    },
+    {
+      "adv1_neg_pct": 0.1952832043170929,
+      "adv2_active_pct": 3.306915044784546,
+      "ce_loss": 0.04980571195483208,
+      "epoch": 0.3829278021539689,
+      "grad_norm": 3.930035352706909,
+      "hpd_loss": 0.015151199884712696,
+      "k1_gt_raw_max": 0.28564560413360596,
+      "k1_gt_raw_mean": 0.01884515956044197,
+      "k1_gt_raw_min": -0.001684204675257206,
+      "k1_gt_raw_neg_pct": 0.1952832043170929,
+      "k1_sample_max": -1.9181127548217773,
+      "k1_sample_mean": -3.2553763389587402,
+      "k1_sample_min": -4.8449602127075195,
+      "k1_sample_neg_pct": 75.20833587646484,
+      "learning_rate": 9.859055934360789e-05,
+      "loss": 0.015151196718215942,
+      "mask3_pct": 3.306915044784546,
+      "neg_student_nll_loss": 0.04980571195483208,
+      "sampled_diff_pct": 3.3092129230499268,
+      "step": 120
+    },
+    {
+      "adv1_neg_pct": 0.3264433741569519,
+      "adv2_active_pct": 2.25433087348938,
+      "ce_loss": 0.04018855467438698,
+      "epoch": 0.4467491025129637,
+      "grad_norm": 0.6639359593391418,
+      "hpd_loss": 0.02619136869907379,
+      "k1_gt_raw_max": 0.24626323580741882,
+      "k1_gt_raw_mean": 0.015010751783847809,
+      "k1_gt_raw_min": -0.004701526835560799,
+      "k1_gt_raw_neg_pct": 0.3264433741569519,
+      "k1_sample_max": -1.902454137802124,
+      "k1_sample_mean": -2.805896282196045,
+      "k1_sample_min": -3.829012393951416,
+      "k1_sample_neg_pct": 71.25,
+      "learning_rate": 9.808029767254437e-05,
+      "loss": 0.026191365718841553,
+      "mask3_pct": 2.25433087348938,
+      "neg_student_nll_loss": 0.04018855467438698,
+      "sampled_diff_pct": 2.2565953731536865,
+      "step": 140
+    },
+    {
+      "adv1_neg_pct": 0.3137112259864807,
+      "adv2_active_pct": 1.3726557493209839,
+      "ce_loss": 0.03148110955953598,
+      "epoch": 0.5105704028719585,
+      "grad_norm": 1.548140525817871,
+      "hpd_loss": 0.03039727918803692,
+      "k1_gt_raw_max": 0.20344290137290955,
+      "k1_gt_raw_mean": 0.009032711386680603,
+      "k1_gt_raw_min": -0.0009857998229563236,
+      "k1_gt_raw_neg_pct": 0.3137112259864807,
+      "k1_sample_max": -2.153831958770752,
+      "k1_sample_mean": -2.81235933303833,
+      "k1_sample_min": -3.547999858856201,
+      "k1_sample_neg_pct": 58.75,
+      "learning_rate": 9.749311981482817e-05,
+      "loss": 0.03039727509021759,
+      "mask3_pct": 1.3726557493209839,
+      "neg_student_nll_loss": 0.03148110955953598,
+      "sampled_diff_pct": 1.3726557493209839,
+      "step": 160
+    },
+    {
+      "adv1_neg_pct": 0.4895250201225281,
+      "adv2_active_pct": 1.1766502857208252,
+      "ce_loss": 0.02162190154194832,
+      "epoch": 0.5743917032309533,
+      "grad_norm": 2.528654098510742,
+      "hpd_loss": 0.011955421417951584,
+      "k1_gt_raw_max": 0.20187309384346008,
+      "k1_gt_raw_mean": 0.008551979437470436,
+      "k1_gt_raw_min": -0.0070910281501710415,
+      "k1_gt_raw_neg_pct": 0.4895250201225281,
+      "k1_sample_max": -1.979987621307373,
+      "k1_sample_mean": -2.5711493492126465,
+      "k1_sample_min": -3.2561821937561035,
+      "k1_sample_neg_pct": 56.09375,
+      "learning_rate": 9.682996608280154e-05,
+      "loss": 0.011955419182777404,
+      "mask3_pct": 1.1766502857208252,
+      "neg_student_nll_loss": 0.02162190154194832,
+      "sampled_diff_pct": 1.185581922531128,
+      "step": 180
+    },
+    {
+      "adv1_neg_pct": 0.5023188591003418,
+      "adv2_active_pct": 1.5687706470489502,
+      "ce_loss": 0.027880553156137466,
+      "epoch": 0.6382130035899481,
+      "grad_norm": 0.9650227427482605,
+      "hpd_loss": 0.010340285487473011,
+      "k1_gt_raw_max": 0.2007327377796173,
+      "k1_gt_raw_mean": 0.009667925536632538,
+      "k1_gt_raw_min": -0.01589183136820793,
+      "k1_gt_raw_neg_pct": 0.5023188591003418,
+      "k1_sample_max": -1.5354162454605103,
+      "k1_sample_mean": -2.1268630027770996,
+      "k1_sample_min": -2.771559238433838,
+      "k1_sample_neg_pct": 56.40625,
+      "learning_rate": 9.609189845731207e-05,
+      "loss": 0.010340283811092376,
+      "mask3_pct": 1.5687706470489502,
+      "neg_student_nll_loss": 0.027880553156137466,
+      "sampled_diff_pct": 1.5725816488265991,
+      "step": 200
+    },
+    {
+      "adv1_neg_pct": 0.48300591111183167,
+      "adv2_active_pct": 1.1448569297790527,
+      "ce_loss": 0.023295864462852478,
+      "epoch": 0.702034303948943,
+      "grad_norm": 2.59177303314209,
+      "hpd_loss": 0.019972147420048714,
+      "k1_gt_raw_max": 0.18090441823005676,
+      "k1_gt_raw_mean": 0.0075406162068247795,
+      "k1_gt_raw_min": -0.014693280681967735,
+      "k1_gt_raw_neg_pct": 0.48300591111183167,
+      "k1_sample_max": -1.6416490077972412,
+      "k1_sample_mean": -2.2296323776245117,
+      "k1_sample_min": -2.930915355682373,
+      "k1_sample_neg_pct": 48.515625,
+      "learning_rate": 9.52800988870462e-05,
+      "loss": 0.01997213959693909,
+      "mask3_pct": 1.1448569297790527,
+      "neg_student_nll_loss": 0.023295864462852478,
+      "sampled_diff_pct": 1.154510736465454,
+      "step": 220
+    },
+    {
+      "adv1_neg_pct": 0.765082597732544,
+      "adv2_active_pct": 0.9244414567947388,
+      "ce_loss": 0.02368738129734993,
+      "epoch": 0.7658556043079378,
+      "grad_norm": 1.1686080694198608,
+      "hpd_loss": 0.02641882374882698,
+      "k1_gt_raw_max": 0.1654866635799408,
+      "k1_gt_raw_mean": 0.005874675698578358,
+      "k1_gt_raw_min": -0.009244760498404503,
+      "k1_gt_raw_neg_pct": 0.765082597732544,
+      "k1_sample_max": -1.3134702444076538,
+      "k1_sample_mean": -1.9339661598205566,
+      "k1_sample_min": -2.6181559562683105,
+      "k1_sample_neg_pct": 45.3125,
+      "learning_rate": 9.439586739574513e-05,
+      "loss": 0.026418817043304444,
+      "mask3_pct": 0.9244414567947388,
+      "neg_student_nll_loss": 0.02368738129734993,
+      "sampled_diff_pct": 0.9244414567947388,
+      "step": 240
+    },
+    {
+      "adv1_neg_pct": 0.8438892364501953,
+      "adv2_active_pct": 0.929736852645874,
+      "ce_loss": 0.020094729959964752,
+      "epoch": 0.8296769046669326,
+      "grad_norm": 1.2779905796051025,
+      "hpd_loss": 0.015618223696947098,
+      "k1_gt_raw_max": 0.16853322088718414,
+      "k1_gt_raw_mean": 0.006190822925418615,
+      "k1_gt_raw_min": -0.009831264615058899,
+      "k1_gt_raw_neg_pct": 0.8438892364501953,
+      "k1_sample_max": -1.3172353506088257,
+      "k1_sample_mean": -1.7796552181243896,
+      "k1_sample_min": -2.261643409729004,
+      "k1_sample_neg_pct": 44.95738983154297,
+      "learning_rate": 9.344062000033414e-05,
+      "loss": 0.015618222951889037,
+      "mask3_pct": 0.9239488840103149,
+      "neg_student_nll_loss": 0.020094729959964752,
+      "sampled_diff_pct": 0.9378187656402588,
+      "step": 260
+    },
+    {
+      "adv1_neg_pct": 0.9945487976074219,
+      "adv2_active_pct": 0.7392160892486572,
+      "ce_loss": 0.015887977555394173,
+      "epoch": 0.8934982050259274,
+      "grad_norm": 0.26237931847572327,
+      "hpd_loss": 0.015452454797923565,
+      "k1_gt_raw_max": 0.14525339007377625,
+      "k1_gt_raw_mean": 0.0046372562646865845,
+      "k1_gt_raw_min": -0.003686224576085806,
+      "k1_gt_raw_neg_pct": 0.9945487976074219,
+      "k1_sample_max": -1.206074595451355,
+      "k1_sample_mean": -1.4332499504089355,
+      "k1_sample_min": -1.671927571296692,
+      "k1_sample_neg_pct": 35.625,
+      "learning_rate": 9.241588644329919e-05,
+      "loss": 0.015452450513839722,
+      "mask3_pct": 0.7392160892486572,
+      "neg_student_nll_loss": 0.015887977555394173,
+      "sampled_diff_pct": 0.7392160892486572,
+      "step": 280
+    },
+    {
+      "adv1_neg_pct": 0.9700182676315308,
+      "adv2_active_pct": 0.6018879413604736,
+      "ce_loss": 0.014513559639453888,
+      "epoch": 0.9573195053849222,
+      "grad_norm": 2.0338079929351807,
+      "hpd_loss": 0.013913260772824287,
+      "k1_gt_raw_max": 0.13846856355667114,
+      "k1_gt_raw_mean": 0.004541991278529167,
+      "k1_gt_raw_min": -0.002533096820116043,
+      "k1_gt_raw_neg_pct": 0.9700182676315308,
+      "k1_sample_max": -1.2417628765106201,
+      "k1_sample_mean": -1.4318351745605469,
+      "k1_sample_min": -1.6163195371627808,
+      "k1_sample_neg_pct": 32.864585876464844,
+      "learning_rate": 9.132330774294252e-05,
+      "loss": 0.013913258910179138,
+      "mask3_pct": 0.6018879413604736,
+      "neg_student_nll_loss": 0.014513559639453888,
+      "sampled_diff_pct": 0.607985258102417,
+      "step": 300
+    },
+    {
+      "adv1_neg_pct": 0.8508870601654053,
+      "adv2_active_pct": 0.859713077545166,
+      "ce_loss": 0.019685909152030945,
+      "epoch": 1.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 86.6822,
+      "eval_generator_rouge1": 99.4006,
+      "eval_generator_rouge2": 99.0209,
+      "eval_generator_rougeL": 98.951,
+      "eval_loss": 0.010082065127789974,
+      "eval_runtime": 595.0313,
+      "eval_samples_per_second": 5.738,
+      "eval_selector_accuracy": 93.9836,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.18,
+      "hpd_loss": 0.02267773076891899,
+      "k1_gt_raw_max": 0.14172008633613586,
+      "k1_gt_raw_mean": 0.0054550133645534515,
+      "k1_gt_raw_min": -0.005414052866399288,
+      "k1_gt_raw_neg_pct": 0.8508870601654053,
+      "k1_sample_max": -1.4249026775360107,
+      "k1_sample_mean": -1.6359399557113647,
+      "k1_sample_min": -1.8393499851226807,
+      "k1_sample_neg_pct": 36.4485969543457,
+      "mask3_pct": 0.859713077545166,
+      "neg_student_nll_loss": 0.019685909152030945,
+      "sampled_diff_pct": 0.859713077545166,
+      "step": 314
+    },
+    {
+      "adv1_neg_pct": 0.6673219203948975,
+      "adv2_active_pct": 1.2173185348510742,
+      "ce_loss": 0.011882005259394646,
+      "epoch": 1.0191463901076985,
+      "grad_norm": 4.386560916900635,
+      "hpd_loss": -0.0019697409588843584,
+      "k1_gt_raw_max": 0.14985336363315582,
+      "k1_gt_raw_mean": 0.006570600904524326,
+      "k1_gt_raw_min": -0.0011412458261474967,
+      "k1_gt_raw_neg_pct": 0.6673219203948975,
+      "k1_sample_max": -0.5733932256698608,
+      "k1_sample_mean": -0.7650949358940125,
+      "k1_sample_min": -0.9706472158432007,
+      "k1_sample_neg_pct": 36.458335876464844,
+      "learning_rate": 9.01646335654399e-05,
+      "loss": 0.014574740827083588,
+      "mask3_pct": 1.2173185348510742,
+      "neg_student_nll_loss": 0.011882005259394646,
+      "sampled_diff_pct": 1.2173185348510742,
+      "step": 320
+    },
+    {
+      "adv1_neg_pct": 1.1466679573059082,
+      "adv2_active_pct": 0.4476942718029022,
+      "ce_loss": 0.009723825380206108,
+      "epoch": 1.0829676904666932,
+      "grad_norm": 1.2898244857788086,
+      "hpd_loss": 0.009684150107204914,
+      "k1_gt_raw_max": 0.10980867594480515,
+      "k1_gt_raw_mean": 0.0032046493142843246,
+      "k1_gt_raw_min": -0.009925473481416702,
+      "k1_gt_raw_neg_pct": 1.1466679573059082,
+      "k1_sample_max": -0.76816326379776,
+      "k1_sample_mean": -0.8626709580421448,
+      "k1_sample_min": -0.9573334455490112,
+      "k1_sample_neg_pct": 28.95833396911621,
+      "learning_rate": 8.894171942290817e-05,
+      "loss": 0.0096841499209404,
+      "mask3_pct": 0.4476942718029022,
+      "neg_student_nll_loss": 0.009723825380206108,
+      "sampled_diff_pct": 0.4520599842071533,
+      "step": 340
+    },
+    {
+      "adv1_neg_pct": 1.657383918762207,
+      "adv2_active_pct": 0.7416136860847473,
+      "ce_loss": 0.01363883912563324,
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.8189982175827026,
+      "hpd_loss": 0.012504098936915398,
+      "k1_gt_raw_max": 0.11255955696105957,
+      "k1_gt_raw_mean": 0.0030380175448954105,
+      "k1_gt_raw_min": -0.01135759986937046,
+      "k1_gt_raw_neg_pct": 1.657383918762207,
+      "k1_sample_max": -1.1351618766784668,
+      "k1_sample_mean": -1.3799619674682617,
+      "k1_sample_min": -1.6486986875534058,
+      "k1_sample_neg_pct": 30.0,
+      "learning_rate": 8.765652370197003e-05,
+      "loss": 0.012504099309444428,
+      "mask3_pct": 0.7416136860847473,
+      "neg_student_nll_loss": 0.01363883912563324,
+      "sampled_diff_pct": 0.7438143491744995,
+      "step": 360
+    },
+    {
+      "adv1_neg_pct": 1.3510011434555054,
+      "adv2_active_pct": 0.3445201814174652,
+      "ce_loss": 0.007902171462774277,
+      "epoch": 1.210610291184683,
+      "grad_norm": 0.5892553329467773,
+      "hpd_loss": 0.008744428865611553,
+      "k1_gt_raw_max": 0.09419713914394379,
+      "k1_gt_raw_mean": 0.0022562379017472267,
+      "k1_gt_raw_min": -0.006403174251317978,
+      "k1_gt_raw_neg_pct": 1.3510011434555054,
+      "k1_sample_max": -0.583869457244873,
+      "k1_sample_mean": -0.7341683506965637,
+      "k1_sample_min": -0.8994753360748291,
+      "k1_sample_neg_pct": 23.75,
+      "learning_rate": 8.631110452757467e-05,
+      "loss": 0.008744429051876067,
+      "mask3_pct": 0.3445201814174652,
+      "neg_student_nll_loss": 0.007902171462774277,
+      "sampled_diff_pct": 0.35682976245880127,
+      "step": 380
+    },
+    {
+      "adv1_neg_pct": 1.4141113758087158,
+      "adv2_active_pct": 0.3374101221561432,
+      "ce_loss": 0.008193380199372768,
+      "epoch": 1.2744315915436777,
+      "grad_norm": 1.2929012775421143,
+      "hpd_loss": 0.006689602509140968,
+      "k1_gt_raw_max": 0.10671139508485794,
+      "k1_gt_raw_mean": 0.002954404801130295,
+      "k1_gt_raw_min": -0.006222248077392578,
+      "k1_gt_raw_neg_pct": 1.4141114950180054,
+      "k1_sample_max": -0.6661227941513062,
+      "k1_sample_mean": -0.7585095763206482,
+      "k1_sample_min": -0.8607568740844727,
+      "k1_sample_neg_pct": 25.760417938232422,
+      "learning_rate": 8.490761646709642e-05,
+      "loss": 0.0066896028816699985,
+      "mask3_pct": 0.3344895839691162,
+      "neg_student_nll_loss": 0.008193380199372768,
+      "sampled_diff_pct": 0.3449546694755554,
+      "step": 400
+    },
+    {
+      "adv1_neg_pct": 2.1220993995666504,
+      "adv2_active_pct": 0.4313049912452698,
+      "ce_loss": 0.011112527921795845,
+      "epoch": 1.3382528919026724,
+      "grad_norm": 0.8164143562316895,
+      "hpd_loss": 0.010808300226926804,
+      "k1_gt_raw_max": 0.10569541901350021,
+      "k1_gt_raw_mean": 0.0035689054057002068,
+      "k1_gt_raw_min": -0.004470471758395433,
+      "k1_gt_raw_neg_pct": 2.1220993995666504,
+      "k1_sample_max": -0.6986867189407349,
+      "k1_sample_mean": -0.7543543577194214,
+      "k1_sample_min": -0.8270018100738525,
+      "k1_sample_neg_pct": 25.279605865478516,
+      "learning_rate": 8.344830707998953e-05,
+      "loss": 0.010808297991752624,
+      "mask3_pct": 0.4313049912452698,
+      "neg_student_nll_loss": 0.011112527921795845,
+      "sampled_diff_pct": 0.4342671036720276,
+      "step": 420
+    },
+    {
+      "adv1_neg_pct": 1.9999275207519531,
+      "adv2_active_pct": 0.2967550456523895,
+      "ce_loss": 0.007173327729105949,
+      "epoch": 1.4020741922616673,
+      "grad_norm": 0.959374189376831,
+      "hpd_loss": 0.005844146944582462,
+      "k1_gt_raw_max": 0.08694229274988174,
+      "k1_gt_raw_mean": 0.002139701507985592,
+      "k1_gt_raw_min": -0.005928537808358669,
+      "k1_gt_raw_neg_pct": 1.9999275207519531,
+      "k1_sample_max": -0.8034830093383789,
+      "k1_sample_mean": -0.8980427980422974,
+      "k1_sample_min": -0.9926026463508606,
+      "k1_sample_neg_pct": 20.78125,
+      "learning_rate": 8.193551331852458e-05,
+      "loss": 0.005844147875905037,
+      "mask3_pct": 0.2967550456523895,
+      "neg_student_nll_loss": 0.007173327729105949,
+      "sampled_diff_pct": 0.30670204758644104,
+      "step": 440
+    },
+    {
+      "adv1_neg_pct": 1.9445667266845703,
+      "adv2_active_pct": 0.28356558084487915,
+      "ce_loss": 0.007787229493260384,
+      "epoch": 1.4658954926206622,
+      "grad_norm": 1.3586090803146362,
+      "hpd_loss": 0.009073459543287754,
+      "k1_gt_raw_max": 0.07425311952829361,
+      "k1_gt_raw_mean": 0.0016224482096731663,
+      "k1_gt_raw_min": -0.0022226697765290737,
+      "k1_gt_raw_neg_pct": 1.9445667266845703,
+      "k1_sample_max": -0.6454492807388306,
+      "k1_sample_mean": -0.686149001121521,
+      "k1_sample_min": -0.7300035953521729,
+      "k1_sample_neg_pct": 17.1875,
+      "learning_rate": 8.037165778537033e-05,
+      "loss": 0.009073460102081298,
+      "mask3_pct": 0.27911925315856934,
+      "neg_student_nll_loss": 0.007787229493260384,
+      "sampled_diff_pct": 0.28356558084487915,
+      "step": 460
+    },
+    {
+      "adv1_neg_pct": 1.8402546644210815,
+      "adv2_active_pct": 0.41872814297676086,
+      "ce_loss": 0.005175907164812088,
+      "epoch": 1.5297167929796571,
+      "grad_norm": 0.2023886740207672,
+      "hpd_loss": 0.0015706174308434129,
+      "k1_gt_raw_max": 0.07684825360774994,
+      "k1_gt_raw_mean": 0.0022208630107343197,
+      "k1_gt_raw_min": -0.008054896257817745,
+      "k1_gt_raw_neg_pct": 1.8402546644210815,
+      "k1_sample_max": -0.47196078300476074,
+      "k1_sample_mean": -0.4973880648612976,
+      "k1_sample_min": -0.5228154063224792,
+      "k1_sample_neg_pct": 17.5,
+      "learning_rate": 7.875924485401427e-05,
+      "loss": 0.0015706172212958335,
+      "mask3_pct": 0.41872814297676086,
+      "neg_student_nll_loss": 0.005175907164812088,
+      "sampled_diff_pct": 0.4208254814147949,
+      "step": 480
+    },
+    {
+      "adv1_neg_pct": 2.5322799682617188,
+      "adv2_active_pct": 0.33805567026138306,
+      "ce_loss": 0.00585869699716568,
+      "epoch": 1.5935380933386518,
+      "grad_norm": 0.030877001583576202,
+      "hpd_loss": 0.003286010818555951,
+      "k1_gt_raw_max": 0.058261577039957047,
+      "k1_gt_raw_mean": 0.002064550295472145,
+      "k1_gt_raw_min": -0.0029995564837008715,
+      "k1_gt_raw_neg_pct": 2.5322799682617188,
+      "k1_sample_max": -0.3366956412792206,
+      "k1_sample_mean": -0.39731863141059875,
+      "k1_sample_min": -0.45794159173965454,
+      "k1_sample_neg_pct": 14.0625,
+      "learning_rate": 7.710085665823444e-05,
+      "loss": 0.0032860103994607924,
+      "mask3_pct": 0.33805567026138306,
+      "neg_student_nll_loss": 0.00585869699716568,
+      "sampled_diff_pct": 0.3419136703014374,
+      "step": 500
+    },
+    {
+      "adv1_neg_pct": 2.701911449432373,
+      "adv2_active_pct": 0.3643760681152344,
+      "ce_loss": 0.010007714852690697,
+      "epoch": 1.6573593936976465,
+      "grad_norm": 0.14292795956134796,
+      "hpd_loss": 0.012855440378189087,
+      "k1_gt_raw_max": 0.06845426559448242,
+      "k1_gt_raw_mean": 0.0017418819479644299,
+      "k1_gt_raw_min": -0.004030767362564802,
+      "k1_gt_raw_neg_pct": 2.701911449432373,
+      "k1_sample_max": -0.4802917242050171,
+      "k1_sample_mean": -0.5421380996704102,
+      "k1_sample_min": -0.6135058999061584,
+      "k1_sample_neg_pct": 17.03125,
+      "learning_rate": 7.539914895704521e-05,
+      "loss": 0.012855446338653565,
+      "mask3_pct": 0.3643760681152344,
+      "neg_student_nll_loss": 0.010007714852690697,
+      "sampled_diff_pct": 0.3710411787033081,
+      "step": 520
+    },
+    {
+      "adv1_neg_pct": 2.248600959777832,
+      "adv2_active_pct": 0.5108850598335266,
+      "ce_loss": 0.01003289595246315,
+      "epoch": 1.7211806940566414,
+      "grad_norm": 0.6681362390518188,
+      "hpd_loss": 0.009245548397302628,
+      "k1_gt_raw_max": 0.08858497440814972,
+      "k1_gt_raw_mean": 0.002853624988347292,
+      "k1_gt_raw_min": -0.004460392985492945,
+      "k1_gt_raw_neg_pct": 2.248600959777832,
+      "k1_sample_max": -0.7408795356750488,
+      "k1_sample_mean": -0.8169400691986084,
+      "k1_sample_min": -0.8906024694442749,
+      "k1_sample_neg_pct": 22.65625,
+      "learning_rate": 7.365684688173908e-05,
+      "loss": 0.009245550632476807,
+      "mask3_pct": 0.5108850598335266,
+      "neg_student_nll_loss": 0.01003289595246315,
+      "sampled_diff_pct": 0.5210334062576294,
+      "step": 540
+    },
+    {
+      "adv1_neg_pct": 2.4635496139526367,
+      "adv2_active_pct": 0.22115065157413483,
+      "ce_loss": 0.005004607141017914,
+      "epoch": 1.7850019944156363,
+      "grad_norm": 0.08243642747402191,
+      "hpd_loss": 0.0062531642615795135,
+      "k1_gt_raw_max": 0.0580272376537323,
+      "k1_gt_raw_mean": 0.0011763360816985369,
+      "k1_gt_raw_min": -0.005644598975777626,
+      "k1_gt_raw_neg_pct": 2.4635493755340576,
+      "k1_sample_max": -0.2404015064239502,
+      "k1_sample_mean": -0.2807300090789795,
+      "k1_sample_min": -0.32241004705429077,
+      "k1_sample_neg_pct": 14.921875,
+      "learning_rate": 7.18767405718347e-05,
+      "loss": 0.006253162026405334,
+      "mask3_pct": 0.22115065157413483,
+      "neg_student_nll_loss": 0.005004607141017914,
+      "sampled_diff_pct": 0.22733174264431,
+      "step": 560
+    },
+    {
+      "adv1_neg_pct": 3.3381173610687256,
+      "adv2_active_pct": 0.23258033394813538,
+      "ce_loss": 0.006839662324637175,
+      "epoch": 1.848823294774631,
+      "grad_norm": 0.0869053527712822,
+      "hpd_loss": 0.00969111267477274,
+      "k1_gt_raw_max": 0.05798381567001343,
+      "k1_gt_raw_mean": 0.0016341875307261944,
+      "k1_gt_raw_min": -0.009443331509828568,
+      "k1_gt_raw_neg_pct": 3.3381173610687256,
+      "k1_sample_max": -0.29867684841156006,
+      "k1_sample_mean": -0.31859445571899414,
+      "k1_sample_min": -0.338512122631073,
+      "k1_sample_neg_pct": 10.625,
+      "learning_rate": 7.006168070692036e-05,
+      "loss": 0.009691111743450165,
+      "mask3_pct": 0.23258033394813538,
+      "neg_student_nll_loss": 0.006839662324637175,
+      "sampled_diff_pct": 0.24274389445781708,
+      "step": 580
+    },
+    {
+      "adv1_neg_pct": 4.040760040283203,
+      "adv2_active_pct": 0.20796719193458557,
+      "ce_loss": 0.002632163930684328,
+      "epoch": 1.9126445951336257,
+      "grad_norm": 0.2512037456035614,
+      "hpd_loss": 0.0005975529784336686,
+      "k1_gt_raw_max": 0.049394719302654266,
+      "k1_gt_raw_mean": 0.0014062562258914113,
+      "k1_gt_raw_min": -0.002836518455296755,
+      "k1_gt_raw_neg_pct": 4.040760040283203,
+      "k1_sample_max": -0.23663434386253357,
+      "k1_sample_mean": -0.255886435508728,
+      "k1_sample_min": -0.2802615463733673,
+      "k1_sample_neg_pct": 10.208333969116211,
+      "learning_rate": 6.821457394154786e-05,
+      "loss": 0.0005975536536425352,
+      "mask3_pct": 0.20796719193458557,
+      "neg_student_nll_loss": 0.002632163930684328,
+      "sampled_diff_pct": 0.21867018938064575,
+      "step": 600
+    },
+    {
+      "adv1_neg_pct": 4.097904205322266,
+      "adv2_active_pct": 0.1848699152469635,
+      "ce_loss": 0.0032025063410401344,
+      "epoch": 1.9764658954926206,
+      "grad_norm": 0.338096022605896,
+      "hpd_loss": 0.002149906475096941,
+      "k1_gt_raw_max": 0.043635837733745575,
+      "k1_gt_raw_mean": 0.0007720005232840776,
+      "k1_gt_raw_min": -0.0036990540102124214,
+      "k1_gt_raw_neg_pct": 4.097904205322266,
+      "k1_sample_max": -0.2395995557308197,
+      "k1_sample_mean": -0.2737259864807129,
+      "k1_sample_min": -0.3078524172306061,
+      "k1_sample_neg_pct": 13.4375,
+      "learning_rate": 6.633837825048768e-05,
+      "loss": 0.0021499061957001688,
+      "mask3_pct": 0.1848699152469635,
+      "neg_student_nll_loss": 0.0032025063410401344,
+      "sampled_diff_pct": 0.18927133083343506,
+      "step": 620
+    },
+    {
+      "adv1_neg_pct": 3.390739917755127,
+      "adv2_active_pct": 0.10791555047035217,
+      "ce_loss": 0.001353576430119574,
+      "epoch": 2.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.1519,
+      "eval_generator_rouge1": 99.7488,
+      "eval_generator_rouge2": 99.6229,
+      "eval_generator_rougeL": 99.5782,
+      "eval_loss": 0.004672760609537363,
+      "eval_runtime": 596.1717,
+      "eval_samples_per_second": 5.727,
+      "eval_selector_accuracy": 98.014,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.179,
+      "hpd_loss": 0.0011793917510658503,
+      "k1_gt_raw_max": 0.03581507131457329,
+      "k1_gt_raw_mean": 0.0006900117732584476,
+      "k1_gt_raw_min": -0.002846459159627557,
+      "k1_gt_raw_neg_pct": 3.390739917755127,
+      "k1_sample_max": -0.06917751580476761,
+      "k1_sample_mean": -0.06961424648761749,
+      "k1_sample_min": -0.07005097717046738,
+      "k1_sample_neg_pct": 10.1694917678833,
+      "mask3_pct": 0.10791555047035217,
+      "neg_student_nll_loss": 0.001353576430119574,
+      "sampled_diff_pct": 0.10791555047035217,
+      "step": 628
+    },
+    {
+      "adv1_neg_pct": 3.476264715194702,
+      "adv2_active_pct": 0.12990054488182068,
+      "ce_loss": 0.004336793441325426,
+      "epoch": 2.038292780215397,
+      "grad_norm": 0.6891085505485535,
+      "hpd_loss": 0.002609923481941223,
+      "k1_gt_raw_max": 0.038067810237407684,
+      "k1_gt_raw_mean": 0.0007896563620306551,
+      "k1_gt_raw_min": -0.005201837047934532,
+      "k1_gt_raw_neg_pct": 3.476264715194702,
+      "k1_sample_max": -0.2407418042421341,
+      "k1_sample_mean": -0.2449159324169159,
+      "k1_sample_min": -0.2522057890892029,
+      "k1_sample_neg_pct": 9.798177719116211,
+      "learning_rate": 6.443609819179926e-05,
+      "loss": 0.0020211638882756235,
+      "mask3_pct": 0.12990054488182068,
+      "neg_student_nll_loss": 0.004336793441325426,
+      "sampled_diff_pct": 0.14956170320510864,
+      "step": 640
+    },
+    {
+      "adv1_neg_pct": 3.8216357231140137,
+      "adv2_active_pct": 0.16393554210662842,
+      "ce_loss": 0.0034592957235872746,
+      "epoch": 2.102114080574392,
+      "grad_norm": 1.311052918434143,
+      "hpd_loss": 0.004151628818362951,
+      "k1_gt_raw_max": 0.04581638425588608,
+      "k1_gt_raw_mean": 0.0008853404433466494,
+      "k1_gt_raw_min": -0.0063157654367387295,
+      "k1_gt_raw_neg_pct": 3.8216357231140137,
+      "k1_sample_max": -0.250868022441864,
+      "k1_sample_mean": -0.30226844549179077,
+      "k1_sample_min": -0.3555181622505188,
+      "k1_sample_neg_pct": 10.833333969116211,
+      "learning_rate": 6.251078009530256e-05,
+      "loss": 0.004151628911495208,
+      "mask3_pct": 0.16393554210662842,
+      "neg_student_nll_loss": 0.0034592957235872746,
+      "sampled_diff_pct": 0.16612085700035095,
+      "step": 660
+    },
+    {
+      "adv1_neg_pct": 4.672894477844238,
+      "adv2_active_pct": 0.17785239219665527,
+      "ce_loss": 0.0037891073152422905,
+      "epoch": 2.1659353809333863,
+      "grad_norm": 0.20577986538410187,
+      "hpd_loss": 0.004049420822411776,
+      "k1_gt_raw_max": 0.047759026288986206,
+      "k1_gt_raw_mean": 0.0010929291602224112,
+      "k1_gt_raw_min": -0.008511744439601898,
+      "k1_gt_raw_neg_pct": 4.672894477844238,
+      "k1_sample_max": -0.2957162857055664,
+      "k1_sample_mean": -0.3683391511440277,
+      "k1_sample_min": -0.440962016582489,
+      "k1_sample_neg_pct": 12.8125,
+      "learning_rate": 6.0565507184155677e-05,
+      "loss": 0.004049421846866607,
+      "mask3_pct": 0.17785239219665527,
+      "neg_student_nll_loss": 0.0037891073152422905,
+      "sampled_diff_pct": 0.17980551719665527,
+      "step": 680
+    },
+    {
+      "adv1_neg_pct": 3.504452705383301,
+      "adv2_active_pct": 0.33364731073379517,
+      "ce_loss": 0.00837249867618084,
+      "epoch": 2.2297566812923812,
+      "grad_norm": 0.05891747772693634,
+      "hpd_loss": 0.012418361380696297,
+      "k1_gt_raw_max": 0.044552117586135864,
+      "k1_gt_raw_mean": 0.001460403436794877,
+      "k1_gt_raw_min": -0.0013761408627033234,
+      "k1_gt_raw_neg_pct": 3.504452705383301,
+      "k1_sample_max": -0.41403040289878845,
+      "k1_sample_mean": -0.4155550003051758,
+      "k1_sample_min": -0.4170795679092407,
+      "k1_sample_neg_pct": 12.8125,
+      "learning_rate": 5.860339463735136e-05,
+      "loss": 0.012418360263109208,
+      "mask3_pct": 0.33364731073379517,
+      "neg_student_nll_loss": 0.00837249867618084,
+      "sampled_diff_pct": 0.33364731073379517,
+      "step": 700
+    },
+    {
+      "adv1_neg_pct": 2.9557998180389404,
+      "adv2_active_pct": 0.2356344759464264,
+      "ce_loss": 0.002732612658292055,
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.03069828823208809,
+      "hpd_loss": 0.0013308569323271513,
+      "k1_gt_raw_max": 0.043037086725234985,
+      "k1_gt_raw_mean": 0.001189029891975224,
+      "k1_gt_raw_min": -0.00335265276953578,
+      "k1_gt_raw_neg_pct": 2.9557998180389404,
+      "k1_sample_max": -0.12545080482959747,
+      "k1_sample_mean": -0.13167737424373627,
+      "k1_sample_min": -0.13885264098644257,
+      "k1_sample_neg_pct": 12.1875,
+      "learning_rate": 5.662758460103895e-05,
+      "loss": 0.0013308576308190823,
+      "mask3_pct": 0.2356344759464264,
+      "neg_student_nll_loss": 0.002732612658292055,
+      "sampled_diff_pct": 0.24056380987167358,
+      "step": 720
+    },
+    {
+      "adv1_neg_pct": 4.362781524658203,
+      "adv2_active_pct": 0.14987114071846008,
+      "ce_loss": 0.00366773409768939,
+      "epoch": 2.357399282010371,
+      "grad_norm": 0.03650576248764992,
+      "hpd_loss": 0.0028897470328956842,
+      "k1_gt_raw_max": 0.03678428381681442,
+      "k1_gt_raw_mean": 0.000713786983396858,
+      "k1_gt_raw_min": -0.007670572958886623,
+      "k1_gt_raw_neg_pct": 4.362782001495361,
+      "k1_sample_max": -0.16289430856704712,
+      "k1_sample_mean": -0.19424015283584595,
+      "k1_sample_min": -0.2255859375,
+      "k1_sample_neg_pct": 9.0625,
+      "learning_rate": 5.464124115666095e-05,
+      "loss": 0.002889746427536011,
+      "mask3_pct": 0.14987114071846008,
+      "neg_student_nll_loss": 0.00366773409768939,
+      "sampled_diff_pct": 0.1667182892560959,
+      "step": 740
+    },
+    {
+      "adv1_neg_pct": 4.3088059425354,
+      "adv2_active_pct": 0.073929063975811,
+      "ce_loss": 0.0018987958319485188,
+      "epoch": 2.421220582369366,
+      "grad_norm": 0.03706243261694908,
+      "hpd_loss": 0.001820039818994701,
+      "k1_gt_raw_max": 0.030663518235087395,
+      "k1_gt_raw_mean": 0.0005116415559314191,
+      "k1_gt_raw_min": -0.003718154039233923,
+      "k1_gt_raw_neg_pct": 4.3088059425354,
+      "k1_sample_max": -0.11870913207530975,
+      "k1_sample_mean": -0.12011738121509552,
+      "k1_sample_min": -0.12152564525604248,
+      "k1_sample_neg_pct": 7.03125,
+      "learning_rate": 5.264754525396212e-05,
+      "loss": 0.0018200401216745376,
+      "mask3_pct": 0.073929063975811,
+      "neg_student_nll_loss": 0.0018987958319485188,
+      "sampled_diff_pct": 0.08056812733411789,
+      "step": 760
+    },
+    {
+      "adv1_neg_pct": 4.6282758712768555,
+      "adv2_active_pct": 0.08583934605121613,
+      "ce_loss": 0.000978614203631878,
+      "epoch": 2.4850418827283605,
+      "grad_norm": 0.0633927583694458,
+      "hpd_loss": 0.0008434242918156087,
+      "k1_gt_raw_max": 0.02713455632328987,
+      "k1_gt_raw_mean": 0.0004272414662409574,
+      "k1_gt_raw_min": -0.006445502862334251,
+      "k1_gt_raw_neg_pct": 4.628275394439697,
+      "k1_sample_max": -0.10302496701478958,
+      "k1_sample_mean": -0.11016625910997391,
+      "k1_sample_min": -0.11730755120515823,
+      "k1_sample_neg_pct": 8.90625,
+      "learning_rate": 5.06496896169857e-05,
+      "loss": 0.0008434242568910122,
+      "mask3_pct": 0.08583934605121613,
+      "neg_student_nll_loss": 0.000978614203631878,
+      "sampled_diff_pct": 0.09829449653625488,
+      "step": 780
+    },
+    {
+      "adv1_neg_pct": 5.204377174377441,
+      "adv2_active_pct": 0.15303727984428406,
+      "ce_loss": 0.004278196021914482,
+      "epoch": 2.5488631830873554,
+      "grad_norm": 0.2113279402256012,
+      "hpd_loss": 0.00520919868722558,
+      "k1_gt_raw_max": 0.0305880606174469,
+      "k1_gt_raw_mean": 0.0005127020995132625,
+      "k1_gt_raw_min": -0.0020859134383499622,
+      "k1_gt_raw_neg_pct": 5.204377174377441,
+      "k1_sample_max": -0.07243171334266663,
+      "k1_sample_mean": -0.08607004582881927,
+      "k1_sample_min": -0.10009756684303284,
+      "k1_sample_neg_pct": 10.0,
+      "learning_rate": 4.865087363121384e-05,
+      "loss": 0.005209198221564293,
+      "mask3_pct": 0.15303727984428406,
+      "neg_student_nll_loss": 0.004278196021914482,
+      "sampled_diff_pct": 0.1585448682308197,
+      "step": 800
+    },
+    {
+      "adv1_neg_pct": 5.268265724182129,
+      "adv2_active_pct": 0.10696102678775787,
+      "ce_loss": 0.003248082473874092,
+      "epoch": 2.6126844834463503,
+      "grad_norm": 0.32251229882240295,
+      "hpd_loss": 0.002392602153122425,
+      "k1_gt_raw_max": 0.03637195751070976,
+      "k1_gt_raw_mean": 0.0010833492269739509,
+      "k1_gt_raw_min": -0.0027411947958171368,
+      "k1_gt_raw_neg_pct": 5.268265724182129,
+      "k1_sample_max": -0.06464927643537521,
+      "k1_sample_mean": -0.06609892845153809,
+      "k1_sample_min": -0.06948131322860718,
+      "k1_sample_neg_pct": 6.826923370361328,
+      "learning_rate": 4.6654298220040673e-05,
+      "loss": 0.0023926014080643656,
+      "mask3_pct": 0.1040404736995697,
+      "neg_student_nll_loss": 0.003248082473874092,
+      "sampled_diff_pct": 0.11661312729120255,
+      "step": 820
+    },
+    {
+      "adv1_neg_pct": 5.766000270843506,
+      "adv2_active_pct": 0.06631548702716827,
+      "ce_loss": 0.0020227693021297455,
+      "epoch": 2.6765057838053448,
+      "grad_norm": 0.2545914947986603,
+      "hpd_loss": 0.002442866563796997,
+      "k1_gt_raw_max": 0.027828488498926163,
+      "k1_gt_raw_mean": 0.0007106282282620668,
+      "k1_gt_raw_min": -0.004438548348844051,
+      "k1_gt_raw_neg_pct": 5.766000270843506,
+      "k1_sample_max": -0.10070931911468506,
+      "k1_sample_mean": -0.10070931911468506,
+      "k1_sample_min": -0.10070931911468506,
+      "k1_sample_neg_pct": 7.1875,
+      "learning_rate": 4.466316071878229e-05,
+      "loss": 0.002442866563796997,
+      "mask3_pct": 0.06631548702716827,
+      "neg_student_nll_loss": 0.0020227693021297455,
+      "sampled_diff_pct": 0.07493838667869568,
+      "step": 840
+    },
+    {
+      "adv1_neg_pct": 6.06646728515625,
+      "adv2_active_pct": 0.10010633617639542,
+      "ce_loss": 0.0019979095086455345,
+      "epoch": 2.74032708416434,
+      "grad_norm": 0.2798786759376526,
+      "hpd_loss": 0.0019781002774834633,
+      "k1_gt_raw_max": 0.0327674075961113,
+      "k1_gt_raw_mean": 0.0005696950247511268,
+      "k1_gt_raw_min": -0.003440997563302517,
+      "k1_gt_raw_neg_pct": 6.06646728515625,
+      "k1_sample_max": -0.08324240148067474,
+      "k1_sample_mean": -0.13795003294944763,
+      "k1_sample_min": -0.1926576793193817,
+      "k1_sample_neg_pct": 9.375,
+      "learning_rate": 4.2680649754432913e-05,
+      "loss": 0.0019781002774834633,
+      "mask3_pct": 0.10010633617639542,
+      "neg_student_nll_loss": 0.0019979095086455345,
+      "sampled_diff_pct": 0.10264699161052704,
+      "step": 860
+    },
+    {
+      "adv1_neg_pct": 5.891697883605957,
+      "adv2_active_pct": 0.07234375178813934,
+      "ce_loss": 0.0014330728445202112,
+      "epoch": 2.8041483845233346,
+      "grad_norm": 0.09429220855236053,
+      "hpd_loss": 0.0010787295177578926,
+      "k1_gt_raw_max": 0.038590773940086365,
+      "k1_gt_raw_mean": 0.0007731987861916423,
+      "k1_gt_raw_min": -0.0038950724992901087,
+      "k1_gt_raw_neg_pct": 5.891697883605957,
+      "k1_sample_max": -0.1224956065416336,
+      "k1_sample_mean": -0.1250840276479721,
+      "k1_sample_min": -0.12767243385314941,
+      "k1_sample_neg_pct": 7.34375,
+      "learning_rate": 4.070994013936646e-05,
+      "loss": 0.001078729424625635,
+      "mask3_pct": 0.07234375178813934,
+      "neg_student_nll_loss": 0.0014330728445202112,
+      "sampled_diff_pct": 0.07657574117183685,
+      "step": 880
+    },
+    {
+      "adv1_neg_pct": 6.011615753173828,
+      "adv2_active_pct": 0.09838910400867462,
+      "ce_loss": 0.0020957919768989086,
+      "epoch": 2.8679696848823295,
+      "grad_norm": 1.0214177370071411,
+      "hpd_loss": 0.0015987120568752289,
+      "k1_gt_raw_max": 0.03688127547502518,
+      "k1_gt_raw_mean": 0.0006160076009109616,
+      "k1_gt_raw_min": -0.005018594674766064,
+      "k1_gt_raw_neg_pct": 6.011615753173828,
+      "k1_sample_max": -0.2794632613658905,
+      "k1_sample_mean": -0.2854064106941223,
+      "k1_sample_min": -0.2913495600223541,
+      "k1_sample_neg_pct": 8.75,
+      "learning_rate": 3.8754187787161155e-05,
+      "loss": 0.0015987120568752289,
+      "mask3_pct": 0.09838910400867462,
+      "neg_student_nll_loss": 0.0020957919768989086,
+      "sampled_diff_pct": 0.10469574481248856,
+      "step": 900
+    },
+    {
+      "adv1_neg_pct": 5.8451714515686035,
+      "adv2_active_pct": 0.12536077201366425,
+      "ce_loss": 0.0031956052407622337,
+      "epoch": 2.9317909852413244,
+      "grad_norm": 0.009789629839360714,
+      "hpd_loss": 0.005076252389699221,
+      "k1_gt_raw_max": 0.03244764730334282,
+      "k1_gt_raw_mean": 0.0005789952119812369,
+      "k1_gt_raw_min": -0.004300437867641449,
+      "k1_gt_raw_neg_pct": 5.8451714515686035,
+      "k1_sample_max": -0.09452944993972778,
+      "k1_sample_mean": -0.1174035519361496,
+      "k1_sample_min": -0.1418294906616211,
+      "k1_sample_neg_pct": 8.4375,
+      "learning_rate": 3.681652465868875e-05,
+      "loss": 0.005076252669095993,
+      "mask3_pct": 0.12536077201366425,
+      "neg_student_nll_loss": 0.0031956052407622337,
+      "sampled_diff_pct": 0.129419207572937,
+      "step": 920
+    },
+    {
+      "adv1_neg_pct": 6.187288284301758,
+      "adv2_active_pct": 0.08470146358013153,
+      "ce_loss": 0.001648780656978488,
+      "epoch": 2.995612285600319,
+      "grad_norm": 0.1643683910369873,
+      "hpd_loss": 0.0010852484265342355,
+      "k1_gt_raw_max": 0.03225347399711609,
+      "k1_gt_raw_mean": 0.0005872219335287809,
+      "k1_gt_raw_min": -0.0068657128140330315,
+      "k1_gt_raw_neg_pct": 6.187288284301758,
+      "k1_sample_max": -0.11387737095355988,
+      "k1_sample_mean": -0.13910287618637085,
+      "k1_sample_min": -0.1643284261226654,
+      "k1_sample_neg_pct": 7.8125,
+      "learning_rate": 3.490005374656184e-05,
+      "loss": 0.0010852484032511712,
+      "mask3_pct": 0.08470146358013153,
+      "neg_student_nll_loss": 0.001648780656978488,
+      "sampled_diff_pct": 0.08656158298254013,
+      "step": 940
+    },
+    {
+      "adv1_neg_pct": 5.000968933105469,
+      "adv2_active_pct": 0.110027015209198,
+      "ce_loss": 0.001415811711922288,
+      "epoch": 3.0,
+      "eval_generator_count": 1712.0,
+      "eval_generator_exact_match": 95.6776,
+      "eval_generator_rouge1": 99.8328,
+      "eval_generator_rouge2": 99.7335,
+      "eval_generator_rougeL": 99.6469,
+      "eval_loss": 0.00400863541290164,
+      "eval_runtime": 591.8273,
+      "eval_samples_per_second": 5.769,
+      "eval_selector_accuracy": 97.7804,
+      "eval_selector_count": 1712.0,
+      "eval_steps_per_second": 0.181,
+      "hpd_loss": 0.0011558050755411386,
+      "k1_gt_raw_max": 0.03289224952459335,
+      "k1_gt_raw_mean": 0.0007148425211198628,
+      "k1_gt_raw_min": -0.0005029054591432214,
+      "k1_gt_raw_neg_pct": 5.000969409942627,
+      "k1_sample_max": -0.4340977072715759,
+      "k1_sample_mean": -0.4340977072715759,
+      "k1_sample_min": -0.4340977072715759,
+      "k1_sample_neg_pct": 9.090909004211426,
+      "mask3_pct": 0.110027015209198,
+      "neg_student_nll_loss": 0.001415811711922288,
+      "sampled_diff_pct": 0.110027015209198,
+      "step": 942
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.524157743353037e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/qwen3/hpd/checkpoint-942/training_args.bin b/qwen3/hpd/checkpoint-942/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1caa0e72cc5edd590e105073725cf4d832684e44
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f04fac21a72f04221af020c3bb88a0dc0ccf53bb7a0c0da21d1ab444937cb9d
+size 7185
diff --git a/qwen3/hpd/checkpoint-942/zero_to_fp32.py b/qwen3/hpd/checkpoint-942/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/hpd/checkpoint-942/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+                                            AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+                                            AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+    autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+    if autoep_layers is None:
+        autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+    if not isinstance(autoep_layers, list):
+        return False
+    return any(
+        isinstance(entry, dict)
+        and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+        for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+    if _has_autoep_zero3_partitioned_metadata(state_dict):
+        raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+                                  "AutoEP expert parameters are partitioned over expert replica groups, so "
+                                  "global data-parallel consolidation would produce incomplete expert tensors. "
+                                  "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+    for file in model_files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+        _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+    # before the expensive optimizer-shard load below.
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/hpd/eval_results.json b/qwen3/hpd/eval_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..506c1341dca9471f4086333bc4921c6a0ca435e4
--- /dev/null
+++ b/qwen3/hpd/eval_results.json
@@ -0,0 +1,14 @@
+{
+    "epoch": 5.0,
+    "eval_generator_count": 1712.0,
+    "eval_generator_exact_match": 96.6121,
+    "eval_generator_rouge1": 99.8409,
+    "eval_generator_rouge2": 99.7633,
+    "eval_generator_rougeL": 99.6798,
+    "eval_loss": 0.0032775094732642174,
+    "eval_runtime": 579.5375,
+    "eval_samples_per_second": 5.891,
+    "eval_selector_accuracy": 98.7734,
+    "eval_selector_count": 1712.0,
+    "eval_steps_per_second": 0.185
+}
\ No newline at end of file
diff --git a/qwen3/hpd/tokenizer.json b/qwen3/hpd/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/hpd/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/hpd/tokenizer_config.json b/qwen3/hpd/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/hpd/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/qwen3/hpd/train_results.json b/qwen3/hpd/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c469fc62622420f601d18744d16b62303b3078b
--- /dev/null
+++ b/qwen3/hpd/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 5.0,
+    "total_flos": 5.871444840310374e+16,
+    "train_loss": 0.018279938655608827,
+    "train_runtime": 8124.6936,
+    "train_samples_per_second": 6.171,
+    "train_steps_per_second": 0.193
+}
\ No newline at end of file