RemasterPhantom — LoRA adapter

lora_adapter/README.md

@@ -0,0 +1,62 @@
+---
+base_model: unsloth/Llama-3.2-1B-Instruct
+library_name: peft
+model_name: remasterphantom-lora
+tags:
+- base_model:adapter:unsloth/Llama-3.2-1B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for remasterphantom-lora
+
+This model is a fine-tuned version of [unsloth/Llama-3.2-1B-Instruct](https://huggingface.co/unsloth/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.21.0
+- TRL: 1.13.0
+- Transformers: 5.17.0
+- Pytorch: 2.14.0
+- Datasets: 5.0.1
+- Tokenizers: 0.23.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

lora_adapter/adapter_config.json

@@ -0,0 +1,51 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "kasa_config": null,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.21.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

lora_adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a37d910a4f15fea01038dd325891944e24ee48e481f8c85466c76d53a832616
+size 45118424

lora_adapter/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

lora_adapter/checkpoint-1398/README.md

@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Llama-3.2-1B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.21.0

lora_adapter/checkpoint-1398/adapter_config.json

@@ -0,0 +1,51 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "kasa_config": null,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.21.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

lora_adapter/checkpoint-1398/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33f33d935e206a1bdc341454815118413899ebb1da1b9108a76481f174178fd8
+size 45118424

lora_adapter/checkpoint-1398/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

lora_adapter/checkpoint-1398/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb60ac93a66bdd98ac69ffaf5eaf3244cbfe5eaf08341cbf24cb164c827cf57e
+size 90359883

lora_adapter/checkpoint-1398/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f1e0d31acc437fbbd16411d0d11d500c5f4dbbc7561671dab7dbf23eb0f2c43
+size 14455

lora_adapter/checkpoint-1398/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7471fbffe52553d737aced82d885ce9f3214d7bfbd1407b61e2b6ab560fdf1ff
+size 1465

lora_adapter/checkpoint-1398/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

lora_adapter/checkpoint-1398/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|finetune_right_pad_id|>",
+  "padding_side": "left",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null
+}

lora_adapter/checkpoint-1398/trainer_state.json

@@ -0,0 +1,1446 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1398,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 3.418184721469879,
+      "epoch": 0.014321518080916577,
+      "grad_norm": 5.165404319763184,
+      "learning_rate": 3e-05,
+      "loss": 4.75037727355957,
+      "mean_token_accuracy": 0.3666964743286371,
+      "num_tokens": 16368.0,
+      "step": 10
+    },
+    {
+      "entropy": 3.8264630317687987,
+      "epoch": 0.028643036161833155,
+      "grad_norm": 2.789987802505493,
+      "learning_rate": 6.333333333333333e-05,
+      "loss": 3.6756973266601562,
+      "mean_token_accuracy": 0.4501046933233738,
+      "num_tokens": 32587.0,
+      "step": 20
+    },
+    {
+      "entropy": 3.0331100687384605,
+      "epoch": 0.04296455424274973,
+      "grad_norm": 2.0175697803497314,
+      "learning_rate": 9.666666666666667e-05,
+      "loss": 2.153616523742676,
+      "mean_token_accuracy": 0.6448496542870998,
+      "num_tokens": 48627.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.7019604802131654,
+      "epoch": 0.05728607232366631,
+      "grad_norm": 1.4288526773452759,
+      "learning_rate": 0.00013000000000000002,
+      "loss": 1.1227694511413575,
+      "mean_token_accuracy": 0.8201614201068879,
+      "num_tokens": 64784.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.8430624332278966,
+      "epoch": 0.07160759040458288,
+      "grad_norm": 0.9607819318771362,
+      "learning_rate": 0.00016333333333333334,
+      "loss": 0.5127146720886231,
+      "mean_token_accuracy": 0.9230180129408836,
+      "num_tokens": 80994.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.3905979799106717,
+      "epoch": 0.08592910848549946,
+      "grad_norm": 0.8198261260986328,
+      "learning_rate": 0.00019666666666666666,
+      "loss": 0.3345895767211914,
+      "mean_token_accuracy": 0.9489256188273429,
+      "num_tokens": 97095.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.3071663906797767,
+      "epoch": 0.10025062656641603,
+      "grad_norm": 2.082451343536377,
+      "learning_rate": 0.00019999036690733865,
+      "loss": 0.29979329109191893,
+      "mean_token_accuracy": 0.9523964017629624,
+      "num_tokens": 113170.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.24958498124033213,
+      "epoch": 0.11457214464733262,
+      "grad_norm": 1.3423985242843628,
+      "learning_rate": 0.00019995706971049468,
+      "loss": 0.19564504623413087,
+      "mean_token_accuracy": 0.9659054979681969,
+      "num_tokens": 129473.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.16129616871476174,
+      "epoch": 0.1288936627282492,
+      "grad_norm": 0.6217024922370911,
+      "learning_rate": 0.00019989999740035867,
+      "loss": 0.1112561583518982,
+      "mean_token_accuracy": 0.9752564527094364,
+      "num_tokens": 145693.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.1593495703302324,
+      "epoch": 0.14321518080916576,
+      "grad_norm": 0.9386972784996033,
+      "learning_rate": 0.00019981916355176472,
+      "loss": 0.124180269241333,
+      "mean_token_accuracy": 0.9724250495433807,
+      "num_tokens": 161885.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.11440805536694824,
+      "epoch": 0.15753669889008234,
+      "grad_norm": 0.49029725790023804,
+      "learning_rate": 0.00019971458739130598,
+      "loss": 0.08359839916229247,
+      "mean_token_accuracy": 0.9790669329464435,
+      "num_tokens": 178194.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.16797136263921858,
+      "epoch": 0.17185821697099893,
+      "grad_norm": 0.4058690071105957,
+      "learning_rate": 0.00019958629379276138,
+      "loss": 0.14625961780548097,
+      "mean_token_accuracy": 0.9697998225688934,
+      "num_tokens": 194281.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.2288114172872156,
+      "epoch": 0.1861797350519155,
+      "grad_norm": 0.43757033348083496,
+      "learning_rate": 0.00019943431327117957,
+      "loss": 0.1752084255218506,
+      "mean_token_accuracy": 0.9670783936977386,
+      "num_tokens": 210353.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.14037263644859194,
+      "epoch": 0.20050125313283207,
+      "grad_norm": 0.4667147696018219,
+      "learning_rate": 0.00019925868197562062,
+      "loss": 0.09937882423400879,
+      "mean_token_accuracy": 0.9777047835290432,
+      "num_tokens": 226307.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.13454449912533165,
+      "epoch": 0.21482277121374865,
+      "grad_norm": 0.5024069547653198,
+      "learning_rate": 0.00019905944168055798,
+      "loss": 0.10526319742202758,
+      "mean_token_accuracy": 0.9757608838379384,
+      "num_tokens": 242366.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.10749808787368238,
+      "epoch": 0.22914428929466524,
+      "grad_norm": 0.3371286392211914,
+      "learning_rate": 0.00019883663977594208,
+      "loss": 0.07259818315505981,
+      "mean_token_accuracy": 0.9807105578482151,
+      "num_tokens": 258533.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.14817415219731628,
+      "epoch": 0.24346580737558182,
+      "grad_norm": 0.6096305251121521,
+      "learning_rate": 0.00019859032925592874,
+      "loss": 0.1135907769203186,
+      "mean_token_accuracy": 0.9745507664978504,
+      "num_tokens": 274810.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.1133763374760747,
+      "epoch": 0.2577873254564984,
+      "grad_norm": 0.41122370958328247,
+      "learning_rate": 0.00019832056870627417,
+      "loss": 0.09238196611404419,
+      "mean_token_accuracy": 0.977765066921711,
+      "num_tokens": 290922.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.07603299776092172,
+      "epoch": 0.272108843537415,
+      "grad_norm": 0.2798531949520111,
+      "learning_rate": 0.00019802742229040025,
+      "loss": 0.06427340507507324,
+      "mean_token_accuracy": 0.9813245065510273,
+      "num_tokens": 307118.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.11896687960252166,
+      "epoch": 0.2864303616183315,
+      "grad_norm": 0.24208419024944305,
+      "learning_rate": 0.00019771095973413293,
+      "loss": 0.10308414697647095,
+      "mean_token_accuracy": 0.9760065227746964,
+      "num_tokens": 323416.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.09079958228394389,
+      "epoch": 0.3007518796992481,
+      "grad_norm": 0.4041318893432617,
+      "learning_rate": 0.00019737125630911784,
+      "loss": 0.06735156178474426,
+      "mean_token_accuracy": 0.9801696293056011,
+      "num_tokens": 339621.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.09143369558732957,
+      "epoch": 0.3150733977801647,
+      "grad_norm": 0.3946863114833832,
+      "learning_rate": 0.00019700839281491662,
+      "loss": 0.07498582005500794,
+      "mean_token_accuracy": 0.9808824859559536,
+      "num_tokens": 355782.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.12254208489321172,
+      "epoch": 0.32939491586108127,
+      "grad_norm": 0.31145262718200684,
+      "learning_rate": 0.00019662245555978846,
+      "loss": 0.09450015425682068,
+      "mean_token_accuracy": 0.976518415659666,
+      "num_tokens": 371844.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.10461308304220437,
+      "epoch": 0.34371643394199786,
+      "grad_norm": 0.3476834297180176,
+      "learning_rate": 0.00019621353634016153,
+      "loss": 0.08276034593582153,
+      "mean_token_accuracy": 0.9795444570481777,
+      "num_tokens": 388015.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.11855657077394426,
+      "epoch": 0.35803795202291444,
+      "grad_norm": 0.32865747809410095,
+      "learning_rate": 0.00019578173241879872,
+      "loss": 0.09439409971237182,
+      "mean_token_accuracy": 0.9776719771325588,
+      "num_tokens": 404196.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.11360202138312162,
+      "epoch": 0.372359470103831,
+      "grad_norm": 0.41493919491767883,
+      "learning_rate": 0.0001953271465016635,
+      "loss": 0.09676467776298522,
+      "mean_token_accuracy": 0.9767774596810341,
+      "num_tokens": 420199.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.1050779873970896,
+      "epoch": 0.3866809881847476,
+      "grad_norm": 0.22015489637851715,
+      "learning_rate": 0.000194849886713491,
+      "loss": 0.07251381278038024,
+      "mean_token_accuracy": 0.9796808525919914,
+      "num_tokens": 436468.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.10228129292372615,
+      "epoch": 0.40100250626566414,
+      "grad_norm": 0.27469080686569214,
+      "learning_rate": 0.00019435006657206997,
+      "loss": 0.09110605716705322,
+      "mean_token_accuracy": 0.9781862519681453,
+      "num_tokens": 452720.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.09573564094025641,
+      "epoch": 0.4153240243465807,
+      "grad_norm": 0.20515574514865875,
+      "learning_rate": 0.00019382780496124253,
+      "loss": 0.06511516571044922,
+      "mean_token_accuracy": 0.9820534713566303,
+      "num_tokens": 468938.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.06887364573776722,
+      "epoch": 0.4296455424274973,
+      "grad_norm": 0.2742563784122467,
+      "learning_rate": 0.00019328322610262703,
+      "loss": 0.053380900621414186,
+      "mean_token_accuracy": 0.9827778674662113,
+      "num_tokens": 485200.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.10576252338942141,
+      "epoch": 0.4439670605084139,
+      "grad_norm": 0.22944959998130798,
+      "learning_rate": 0.00019271645952607163,
+      "loss": 0.09060812592506409,
+      "mean_token_accuracy": 0.9780363194644451,
+      "num_tokens": 501474.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.1117024946026504,
+      "epoch": 0.4582885785893305,
+      "grad_norm": 0.2669987678527832,
+      "learning_rate": 0.0001921276400388451,
+      "loss": 0.07678983211517335,
+      "mean_token_accuracy": 0.9798587985336781,
+      "num_tokens": 517580.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.10053576473146678,
+      "epoch": 0.47261009667024706,
+      "grad_norm": 0.24858009815216064,
+      "learning_rate": 0.00019151690769357265,
+      "loss": 0.08534708023071289,
+      "mean_token_accuracy": 0.9798071250319481,
+      "num_tokens": 533788.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.13230610005557536,
+      "epoch": 0.48693161475116364,
+      "grad_norm": 0.2949056923389435,
+      "learning_rate": 0.00019088440775492384,
+      "loss": 0.09592426419258118,
+      "mean_token_accuracy": 0.9757725939154624,
+      "num_tokens": 549966.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.06407897693570704,
+      "epoch": 0.5012531328320802,
+      "grad_norm": 0.24437105655670166,
+      "learning_rate": 0.00019023029066506087,
+      "loss": 0.05607050061225891,
+      "mean_token_accuracy": 0.9830520637333393,
+      "num_tokens": 566066.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.06379525621887297,
+      "epoch": 0.5155746509129968,
+      "grad_norm": 0.2406875491142273,
+      "learning_rate": 0.00018955471200785551,
+      "loss": 0.05550529956817627,
+      "mean_token_accuracy": 0.9826574549078941,
+      "num_tokens": 582036.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.10768530536442995,
+      "epoch": 0.5298961689939133,
+      "grad_norm": 0.19389888644218445,
+      "learning_rate": 0.00018885783247188305,
+      "loss": 0.09576615095138549,
+      "mean_token_accuracy": 0.9779540851712227,
+      "num_tokens": 598182.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.12297249888069928,
+      "epoch": 0.54421768707483,
+      "grad_norm": 0.33548423647880554,
+      "learning_rate": 0.0001881398178122017,
+      "loss": 0.08422422409057617,
+      "mean_token_accuracy": 0.9791318334639072,
+      "num_tokens": 614468.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.06059104287996888,
+      "epoch": 0.5585392051557465,
+      "grad_norm": 0.34327495098114014,
+      "learning_rate": 0.0001874008388109276,
+      "loss": 0.05207563042640686,
+      "mean_token_accuracy": 0.9843599937856198,
+      "num_tokens": 630800.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.09479554321151226,
+      "epoch": 0.572860723236663,
+      "grad_norm": 0.3410899043083191,
+      "learning_rate": 0.00018664107123661337,
+      "loss": 0.0662381649017334,
+      "mean_token_accuracy": 0.9813465528190136,
+      "num_tokens": 646956.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.09436679913196713,
+      "epoch": 0.5871822413175797,
+      "grad_norm": 0.8792180418968201,
+      "learning_rate": 0.00018586069580244128,
+      "loss": 0.07904249429702759,
+      "mean_token_accuracy": 0.9798643849790096,
+      "num_tokens": 663240.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.09691147103440016,
+      "epoch": 0.6015037593984962,
+      "grad_norm": 0.23540320992469788,
+      "learning_rate": 0.00018505989812323993,
+      "loss": 0.06907619833946228,
+      "mean_token_accuracy": 0.9815059810876846,
+      "num_tokens": 679484.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.07770795098040253,
+      "epoch": 0.6158252774794128,
+      "grad_norm": 0.28902187943458557,
+      "learning_rate": 0.0001842388686713351,
+      "loss": 0.05818814635276794,
+      "mean_token_accuracy": 0.9836779788136483,
+      "num_tokens": 695720.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.06032483666203916,
+      "epoch": 0.6301467955603294,
+      "grad_norm": 0.328427791595459,
+      "learning_rate": 0.0001833978027312453,
+      "loss": 0.053039717674255374,
+      "mean_token_accuracy": 0.9840192586183548,
+      "num_tokens": 711945.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.0849465676350519,
+      "epoch": 0.644468313641246,
+      "grad_norm": 0.16869916021823883,
+      "learning_rate": 0.00018253690035323285,
+      "loss": 0.06781792044639587,
+      "mean_token_accuracy": 0.9806929357349873,
+      "num_tokens": 728029.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.08969353097490966,
+      "epoch": 0.6587898317221625,
+      "grad_norm": 0.18590566515922546,
+      "learning_rate": 0.0001816563663057211,
+      "loss": 0.06440567970275879,
+      "mean_token_accuracy": 0.9825256459414959,
+      "num_tokens": 744330.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.06217158674262464,
+      "epoch": 0.6731113498030791,
+      "grad_norm": 0.27869918942451477,
+      "learning_rate": 0.00018075641002658976,
+      "loss": 0.053045254945755,
+      "mean_token_accuracy": 0.9843784637749196,
+      "num_tokens": 760323.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.0674417833564803,
+      "epoch": 0.6874328678839957,
+      "grad_norm": 0.1830916553735733,
+      "learning_rate": 0.00017983724557335925,
+      "loss": 0.055755794048309326,
+      "mean_token_accuracy": 0.9838082902133465,
+      "num_tokens": 776628.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.0802982683526352,
+      "epoch": 0.7017543859649122,
+      "grad_norm": 0.4552997350692749,
+      "learning_rate": 0.00017889909157227653,
+      "loss": 0.060858166217803954,
+      "mean_token_accuracy": 0.9824069820344448,
+      "num_tokens": 792838.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.07723174102138727,
+      "epoch": 0.7160759040458289,
+      "grad_norm": 0.18987013399600983,
+      "learning_rate": 0.00017794217116631417,
+      "loss": 0.060531008243560794,
+      "mean_token_accuracy": 0.9814355976879596,
+      "num_tokens": 808999.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.05520166652277112,
+      "epoch": 0.7303974221267454,
+      "grad_norm": 0.18790723383426666,
+      "learning_rate": 0.00017696671196209496,
+      "loss": 0.05018657445907593,
+      "mean_token_accuracy": 0.983309705555439,
+      "num_tokens": 825062.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.05418659960851073,
+      "epoch": 0.744718940207662,
+      "grad_norm": 0.160777747631073,
+      "learning_rate": 0.00017597294597575507,
+      "loss": 0.04926420748233795,
+      "mean_token_accuracy": 0.9841075465083122,
+      "num_tokens": 841241.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.1152086689369753,
+      "epoch": 0.7590404582885786,
+      "grad_norm": 0.6535390615463257,
+      "learning_rate": 0.0001749611095777581,
+      "loss": 0.0968184232711792,
+      "mean_token_accuracy": 0.975805052369833,
+      "num_tokens": 857300.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.07658788892440498,
+      "epoch": 0.7733619763694952,
+      "grad_norm": 0.1728331297636032,
+      "learning_rate": 0.00017393144343667364,
+      "loss": 0.06037890911102295,
+      "mean_token_accuracy": 0.9823231592774391,
+      "num_tokens": 873434.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.09907218038570136,
+      "epoch": 0.7876834944504117,
+      "grad_norm": 0.22389887273311615,
+      "learning_rate": 0.00017288419246193373,
+      "loss": 0.07112311124801636,
+      "mean_token_accuracy": 0.9803691074252129,
+      "num_tokens": 889567.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.05675842456985265,
+      "epoch": 0.8020050125313283,
+      "grad_norm": 0.17525967955589294,
+      "learning_rate": 0.00017181960574557992,
+      "loss": 0.04961739480495453,
+      "mean_token_accuracy": 0.9838187299668789,
+      "num_tokens": 905625.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.08107102129142732,
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.17234288156032562,
+      "learning_rate": 0.00017073793650301624,
+      "loss": 0.06352313160896302,
+      "mean_token_accuracy": 0.9820462837815285,
+      "num_tokens": 921757.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.061689483653754,
+      "epoch": 0.8306480486931614,
+      "grad_norm": 0.23366186022758484,
+      "learning_rate": 0.00016963944201278097,
+      "loss": 0.051079404354095456,
+      "mean_token_accuracy": 0.982898436486721,
+      "num_tokens": 937984.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.06318069302942604,
+      "epoch": 0.8449695667740781,
+      "grad_norm": 0.19096462428569794,
+      "learning_rate": 0.00016852438355535214,
+      "loss": 0.05602290034294129,
+      "mean_token_accuracy": 0.9843950115144253,
+      "num_tokens": 954323.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.05533521031029522,
+      "epoch": 0.8592910848549946,
+      "grad_norm": 0.21601422131061554,
+      "learning_rate": 0.00016739302635100108,
+      "loss": 0.04836077392101288,
+      "mean_token_accuracy": 0.9856260627508163,
+      "num_tokens": 970505.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.05332150736358017,
+      "epoch": 0.8736126029359113,
+      "grad_norm": 0.16544337570667267,
+      "learning_rate": 0.00016624563949670904,
+      "loss": 0.04835851788520813,
+      "mean_token_accuracy": 0.9843276873230934,
+      "num_tokens": 986682.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.052138336235657334,
+      "epoch": 0.8879341210168278,
+      "grad_norm": 0.1802208125591278,
+      "learning_rate": 0.0001650824959021614,
+      "loss": 0.05234106183052063,
+      "mean_token_accuracy": 0.9847004011273384,
+      "num_tokens": 1002790.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.055844594305381176,
+      "epoch": 0.9022556390977443,
+      "grad_norm": 0.15213149785995483,
+      "learning_rate": 0.00016390387222483537,
+      "loss": 0.04984959363937378,
+      "mean_token_accuracy": 0.983958077430725,
+      "num_tokens": 1019089.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.06128789323847741,
+      "epoch": 0.916577157178661,
+      "grad_norm": 0.12318123877048492,
+      "learning_rate": 0.00016271004880419608,
+      "loss": 0.05124086737632751,
+      "mean_token_accuracy": 0.9843113601207734,
+      "num_tokens": 1035332.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.0652699463069439,
+      "epoch": 0.9308986752595775,
+      "grad_norm": 0.16328729689121246,
+      "learning_rate": 0.00016150130959501687,
+      "loss": 0.051763099431991574,
+      "mean_token_accuracy": 0.9837749816477299,
+      "num_tokens": 1051331.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.0749959686305374,
+      "epoch": 0.9452201933404941,
+      "grad_norm": 0.2559461295604706,
+      "learning_rate": 0.0001602779420998398,
+      "loss": 0.05571107268333435,
+      "mean_token_accuracy": 0.9834148533642292,
+      "num_tokens": 1067608.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.05268463878892362,
+      "epoch": 0.9595417114214106,
+      "grad_norm": 0.14703021943569183,
+      "learning_rate": 0.00015904023730059228,
+      "loss": 0.04806509017944336,
+      "mean_token_accuracy": 0.9848555572330951,
+      "num_tokens": 1083704.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.08527488259132951,
+      "epoch": 0.9738632295023273,
+      "grad_norm": 0.27216821908950806,
+      "learning_rate": 0.00015778848958937586,
+      "loss": 0.061145973205566403,
+      "mean_token_accuracy": 0.9831270687282085,
+      "num_tokens": 1099812.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.05362487006932497,
+      "epoch": 0.9881847475832438,
+      "grad_norm": 0.20429278910160065,
+      "learning_rate": 0.00015652299669844438,
+      "loss": 0.04693828225135803,
+      "mean_token_accuracy": 0.9854582466185093,
+      "num_tokens": 1116039.0,
+      "step": 690
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 0.05509232616267439,
+      "eval_loss": 0.04924684017896652,
+      "eval_mean_token_accuracy": 0.9852213520987504,
+      "eval_num_tokens": 1129380.0,
+      "eval_runtime": 132.3751,
+      "eval_samples_per_second": 2.221,
+      "eval_steps_per_second": 2.221,
+      "step": 699
+    },
+    {
+      "entropy": 0.06238776402598297,
+      "epoch": 1.0014321518080918,
+      "grad_norm": 0.12905091047286987,
+      "learning_rate": 0.00015524405962938714,
+      "loss": 0.05233152508735657,
+      "mean_token_accuracy": 0.9846261304778021,
+      "num_tokens": 1130996.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.058638660283759234,
+      "epoch": 1.0157536698890082,
+      "grad_norm": 0.2219594568014145,
+      "learning_rate": 0.00015395198258153498,
+      "loss": 0.05078426599502563,
+      "mean_token_accuracy": 0.9838196195662021,
+      "num_tokens": 1147116.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.05818793845828622,
+      "epoch": 1.0300751879699248,
+      "grad_norm": 0.6569804549217224,
+      "learning_rate": 0.00015264707287960521,
+      "loss": 0.05066579580307007,
+      "mean_token_accuracy": 0.9846140623092652,
+      "num_tokens": 1163282.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.054456926276907326,
+      "epoch": 1.0443967060508415,
+      "grad_norm": 0.19272315502166748,
+      "learning_rate": 0.00015132964090060357,
+      "loss": 0.048507091403007505,
+      "mean_token_accuracy": 0.9837053954601288,
+      "num_tokens": 1179237.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.04848019825294614,
+      "epoch": 1.0587182241317579,
+      "grad_norm": 0.1253335028886795,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 0.04691898822784424,
+      "mean_token_accuracy": 0.9845252685248852,
+      "num_tokens": 1195531.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.06931644980795681,
+      "epoch": 1.0730397422126745,
+      "grad_norm": 0.19210802018642426,
+      "learning_rate": 0.0001486584664371961,
+      "loss": 0.05346042513847351,
+      "mean_token_accuracy": 0.9840465135872364,
+      "num_tokens": 1212061.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.0602082708152011,
+      "epoch": 1.0873612602935911,
+      "grad_norm": 0.18707653880119324,
+      "learning_rate": 0.00014730535930030172,
+      "loss": 0.04904778599739075,
+      "mean_token_accuracy": 0.9847656920552254,
+      "num_tokens": 1228125.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.057827019784599545,
+      "epoch": 1.1016827783745078,
+      "grad_norm": 0.1590399444103241,
+      "learning_rate": 0.00014594100043023887,
+      "loss": 0.048907867074012755,
+      "mean_token_accuracy": 0.9835596241056919,
+      "num_tokens": 1244380.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.05155087348539382,
+      "epoch": 1.1160042964554242,
+      "grad_norm": 0.13960455358028412,
+      "learning_rate": 0.00014456571434419074,
+      "loss": 0.04657362103462219,
+      "mean_token_accuracy": 0.9841499179601669,
+      "num_tokens": 1260560.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.0582313765771687,
+      "epoch": 1.1303258145363408,
+      "grad_norm": 0.1564759463071823,
+      "learning_rate": 0.0001431798281584144,
+      "loss": 0.051281589269638064,
+      "mean_token_accuracy": 0.9844076797366142,
+      "num_tokens": 1276932.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.051610140060074625,
+      "epoch": 1.1446473326172575,
+      "grad_norm": 0.16707740724086761,
+      "learning_rate": 0.0001417836715104349,
+      "loss": 0.04861922562122345,
+      "mean_token_accuracy": 0.9845031477510929,
+      "num_tokens": 1293128.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.05285086496733129,
+      "epoch": 1.158968850698174,
+      "grad_norm": 0.13717925548553467,
+      "learning_rate": 0.00014037757648064018,
+      "loss": 0.04775188565254211,
+      "mean_token_accuracy": 0.9854285247623921,
+      "num_tokens": 1309523.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.05543681618291885,
+      "epoch": 1.1732903687790905,
+      "grad_norm": 0.15092015266418457,
+      "learning_rate": 0.00013896187751329437,
+      "loss": 0.046713563799858096,
+      "mean_token_accuracy": 0.9847700297832489,
+      "num_tokens": 1325813.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.04951737595256418,
+      "epoch": 1.1876118868600072,
+      "grad_norm": 0.1319475620985031,
+      "learning_rate": 0.00013753691133698944,
+      "loss": 0.045718804001808167,
+      "mean_token_accuracy": 0.9848351627588272,
+      "num_tokens": 1342049.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.05453478880226612,
+      "epoch": 1.2019334049409238,
+      "grad_norm": 0.1564560830593109,
+      "learning_rate": 0.000136103016884553,
+      "loss": 0.048373016715049746,
+      "mean_token_accuracy": 0.9833561331033707,
+      "num_tokens": 1358086.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.05261064667720348,
+      "epoch": 1.2162549230218402,
+      "grad_norm": 0.1645234227180481,
+      "learning_rate": 0.00013466053521243214,
+      "loss": 0.047138947248458865,
+      "mean_token_accuracy": 0.9853625297546387,
+      "num_tokens": 1373958.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.04612366235814989,
+      "epoch": 1.2305764411027569,
+      "grad_norm": 0.13810141384601593,
+      "learning_rate": 0.000133209809419572,
+      "loss": 0.04452601969242096,
+      "mean_token_accuracy": 0.9859118059277534,
+      "num_tokens": 1390120.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.048122103605419395,
+      "epoch": 1.2448979591836735,
+      "grad_norm": 0.16478563845157623,
+      "learning_rate": 0.00013175118456580857,
+      "loss": 0.04704745411872864,
+      "mean_token_accuracy": 0.9848096296191216,
+      "num_tokens": 1406408.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.05111220530234277,
+      "epoch": 1.25921947726459,
+      "grad_norm": 0.14047689735889435,
+      "learning_rate": 0.00013028500758979506,
+      "loss": 0.04449088275432587,
+      "mean_token_accuracy": 0.9855061650276185,
+      "num_tokens": 1422673.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.050457812706008555,
+      "epoch": 1.2735409953455066,
+      "grad_norm": 0.14891056716442108,
+      "learning_rate": 0.0001288116272264816,
+      "loss": 0.0456391453742981,
+      "mean_token_accuracy": 0.9854636572301387,
+      "num_tokens": 1438953.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.04946525213308632,
+      "epoch": 1.2878625134264232,
+      "grad_norm": 0.16796022653579712,
+      "learning_rate": 0.00012733139392416722,
+      "loss": 0.045487576723098756,
+      "mean_token_accuracy": 0.9840421840548516,
+      "num_tokens": 1455012.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.049669814435765146,
+      "epoch": 1.3021840315073399,
+      "grad_norm": 0.15355756878852844,
+      "learning_rate": 0.0001258446597611447,
+      "loss": 0.04645189344882965,
+      "mean_token_accuracy": 0.9850803188979625,
+      "num_tokens": 1471054.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.048828864190727475,
+      "epoch": 1.3165055495882563,
+      "grad_norm": 0.1641317456960678,
+      "learning_rate": 0.00012435177836195776,
+      "loss": 0.04594441056251526,
+      "mean_token_accuracy": 0.9857151001691818,
+      "num_tokens": 1487139.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.04583641842473298,
+      "epoch": 1.330827067669173,
+      "grad_norm": 0.15289618074893951,
+      "learning_rate": 0.0001228531048132901,
+      "loss": 0.04497186541557312,
+      "mean_token_accuracy": 0.9860778227448463,
+      "num_tokens": 1503603.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.0469824765343219,
+      "epoch": 1.3451485857500896,
+      "grad_norm": 0.13089172542095184,
+      "learning_rate": 0.00012134899557950698,
+      "loss": 0.04497265219688416,
+      "mean_token_accuracy": 0.9844655573368073,
+      "num_tokens": 1519909.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.05958397570066154,
+      "epoch": 1.359470103831006,
+      "grad_norm": 0.3665521442890167,
+      "learning_rate": 0.000119839808417869,
+      "loss": 0.04814833104610443,
+      "mean_token_accuracy": 0.9847050316631794,
+      "num_tokens": 1536192.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.05206554315518588,
+      "epoch": 1.3737916219119226,
+      "grad_norm": 0.15439002215862274,
+      "learning_rate": 0.00011832590229343835,
+      "loss": 0.045511645078659055,
+      "mean_token_accuracy": 0.9845592580735684,
+      "num_tokens": 1552241.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.05069404630921781,
+      "epoch": 1.3881131399928393,
+      "grad_norm": 0.1370713859796524,
+      "learning_rate": 0.00011680763729369783,
+      "loss": 0.04533308446407318,
+      "mean_token_accuracy": 0.9854350961744786,
+      "num_tokens": 1568661.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.048878395347855985,
+      "epoch": 1.402434658073756,
+      "grad_norm": 0.15589790046215057,
+      "learning_rate": 0.00011528537454290294,
+      "loss": 0.046069884300231935,
+      "mean_token_accuracy": 0.9837729744613171,
+      "num_tokens": 1585086.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.045904085785150525,
+      "epoch": 1.4167561761546725,
+      "grad_norm": 0.14884407818317413,
+      "learning_rate": 0.00011375947611618716,
+      "loss": 0.04572470188140869,
+      "mean_token_accuracy": 0.9846699081361294,
+      "num_tokens": 1601155.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.056870630197227004,
+      "epoch": 1.431077694235589,
+      "grad_norm": 0.21461369097232819,
+      "learning_rate": 0.00011223030495344127,
+      "loss": 0.05038630962371826,
+      "mean_token_accuracy": 0.9850619569420814,
+      "num_tokens": 1617145.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.04888233456294984,
+      "epoch": 1.4453992123165056,
+      "grad_norm": 0.13056223094463348,
+      "learning_rate": 0.00011069822477298706,
+      "loss": 0.0462115079164505,
+      "mean_token_accuracy": 0.9843272507190705,
+      "num_tokens": 1633412.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.05794957473408431,
+      "epoch": 1.459720730397422,
+      "grad_norm": 0.3261290490627289,
+      "learning_rate": 0.0001091635999850655,
+      "loss": 0.04766329824924469,
+      "mean_token_accuracy": 0.9835515014827252,
+      "num_tokens": 1649727.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.048225057567469774,
+      "epoch": 1.4740422484783386,
+      "grad_norm": 0.130174919962883,
+      "learning_rate": 0.00010762679560516067,
+      "loss": 0.04580983817577362,
+      "mean_token_accuracy": 0.9854344628751278,
+      "num_tokens": 1665979.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.061461174837313594,
+      "epoch": 1.4883637665592553,
+      "grad_norm": 0.1320205181837082,
+      "learning_rate": 0.00010608817716717976,
+      "loss": 0.048619455099105834,
+      "mean_token_accuracy": 0.9839078761637211,
+      "num_tokens": 1682011.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.052603658544830977,
+      "epoch": 1.502685284640172,
+      "grad_norm": 0.16996844112873077,
+      "learning_rate": 0.00010454811063650957,
+      "loss": 0.04834533929824829,
+      "mean_token_accuracy": 0.9843780644237995,
+      "num_tokens": 1697914.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.054306549951434135,
+      "epoch": 1.5170068027210886,
+      "grad_norm": 0.1440720409154892,
+      "learning_rate": 0.0001030069623229704,
+      "loss": 0.04781841635704041,
+      "mean_token_accuracy": 0.9854689165949821,
+      "num_tokens": 1714070.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.05636612819507718,
+      "epoch": 1.531328320802005,
+      "grad_norm": 0.14176875352859497,
+      "learning_rate": 0.00010146509879368808,
+      "loss": 0.04686476588249207,
+      "mean_token_accuracy": 0.9847516976296902,
+      "num_tokens": 1730131.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.05093875625170767,
+      "epoch": 1.5456498388829216,
+      "grad_norm": 0.1596251130104065,
+      "learning_rate": 9.99228867859049e-05,
+      "loss": 0.04491396546363831,
+      "mean_token_accuracy": 0.9852274671196938,
+      "num_tokens": 1746342.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.04888144808355719,
+      "epoch": 1.559971356963838,
+      "grad_norm": 0.20582543313503265,
+      "learning_rate": 9.838069311974986e-05,
+      "loss": 0.0455554723739624,
+      "mean_token_accuracy": 0.9859486259520054,
+      "num_tokens": 1762591.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.05331454309634864,
+      "epoch": 1.5742928750447547,
+      "grad_norm": 0.1273772269487381,
+      "learning_rate": 9.683888461098933e-05,
+      "loss": 0.04549239873886109,
+      "mean_token_accuracy": 0.9847855247557163,
+      "num_tokens": 1778625.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.048709102207794785,
+      "epoch": 1.5886143931256713,
+      "grad_norm": 0.11710993200540543,
+      "learning_rate": 9.529782798377877e-05,
+      "loss": 0.047628313302993774,
+      "mean_token_accuracy": 0.9854380682110786,
+      "num_tokens": 1794731.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.04785530364606529,
+      "epoch": 1.602935911206588,
+      "grad_norm": 0.1247224509716034,
+      "learning_rate": 9.375788978343608e-05,
+      "loss": 0.04518147110939026,
+      "mean_token_accuracy": 0.9852628707885742,
+      "num_tokens": 1810816.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.05034571662545204,
+      "epoch": 1.6172574292875046,
+      "grad_norm": 0.14023028314113617,
+      "learning_rate": 9.221943628925789e-05,
+      "loss": 0.04668688774108887,
+      "mean_token_accuracy": 0.9848854929208756,
+      "num_tokens": 1826931.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.051596777117811146,
+      "epoch": 1.631578947368421,
+      "grad_norm": 0.1819605678319931,
+      "learning_rate": 9.068283342739853e-05,
+      "loss": 0.04736668169498444,
+      "mean_token_accuracy": 0.9847092233598232,
+      "num_tokens": 1842987.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.052112107467837634,
+      "epoch": 1.6459004654493377,
+      "grad_norm": 0.16193711757659912,
+      "learning_rate": 8.914844668383339e-05,
+      "loss": 0.046776899695396425,
+      "mean_token_accuracy": 0.9839866004884243,
+      "num_tokens": 1858881.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.04619899494573474,
+      "epoch": 1.660221983530254,
+      "grad_norm": 0.16279025375843048,
+      "learning_rate": 8.7616641017427e-05,
+      "loss": 0.04623554050922394,
+      "mean_token_accuracy": 0.9846173226833344,
+      "num_tokens": 1874895.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.047261204943060874,
+      "epoch": 1.6745435016111707,
+      "grad_norm": 0.1297980099916458,
+      "learning_rate": 8.608778077312633e-05,
+      "loss": 0.0437994509935379,
+      "mean_token_accuracy": 0.9848140060901642,
+      "num_tokens": 1891143.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.05167301250621677,
+      "epoch": 1.6888650196920874,
+      "grad_norm": 0.15251320600509644,
+      "learning_rate": 8.456222959530008e-05,
+      "loss": 0.045985713601112366,
+      "mean_token_accuracy": 0.9834124945104122,
+      "num_tokens": 1907142.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.04752049937378615,
+      "epoch": 1.703186537773004,
+      "grad_norm": 0.18589235842227936,
+      "learning_rate": 8.304035034124484e-05,
+      "loss": 0.04648590981960297,
+      "mean_token_accuracy": 0.9851418137550354,
+      "num_tokens": 1923277.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.048138466640375556,
+      "epoch": 1.7175080558539206,
+      "grad_norm": 0.16602511703968048,
+      "learning_rate": 8.152250499487816e-05,
+      "loss": 0.04516423046588898,
+      "mean_token_accuracy": 0.9846760742366314,
+      "num_tokens": 1939220.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.05131866903975606,
+      "epoch": 1.731829573934837,
+      "grad_norm": 0.20442146062850952,
+      "learning_rate": 8.000905458063957e-05,
+      "loss": 0.045997625589370726,
+      "mean_token_accuracy": 0.9854269422590732,
+      "num_tokens": 1955362.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.050703935651108624,
+      "epoch": 1.7461510920157537,
+      "grad_norm": 0.1469806581735611,
+      "learning_rate": 7.850035907761973e-05,
+      "loss": 0.045433256030082705,
+      "mean_token_accuracy": 0.9851394325494767,
+      "num_tokens": 1971476.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.04811225512530655,
+      "epoch": 1.76047261009667,
+      "grad_norm": 0.12229132652282715,
+      "learning_rate": 7.699677733393826e-05,
+      "loss": 0.04342373609542847,
+      "mean_token_accuracy": 0.9854144871234893,
+      "num_tokens": 1987751.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.046569342049770056,
+      "epoch": 1.7747941281775867,
+      "grad_norm": 0.14311222732067108,
+      "learning_rate": 7.549866698139048e-05,
+      "loss": 0.04468758106231689,
+      "mean_token_accuracy": 0.9856827527284622,
+      "num_tokens": 2003903.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.04997451878152788,
+      "epoch": 1.7891156462585034,
+      "grad_norm": 0.1998521387577057,
+      "learning_rate": 7.400638435038371e-05,
+      "loss": 0.04490821659564972,
+      "mean_token_accuracy": 0.9860882505774498,
+      "num_tokens": 2020064.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.047584796766750516,
+      "epoch": 1.80343716433942,
+      "grad_norm": 0.1390482783317566,
+      "learning_rate": 7.252028438518266e-05,
+      "loss": 0.044057485461235044,
+      "mean_token_accuracy": 0.9860201634466648,
+      "num_tokens": 2036074.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.046523316064849496,
+      "epoch": 1.8177586824203367,
+      "grad_norm": 0.15581190586090088,
+      "learning_rate": 7.1040720559485e-05,
+      "loss": 0.044152289628982544,
+      "mean_token_accuracy": 0.984954721480608,
+      "num_tokens": 2052410.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.048715499113313854,
+      "epoch": 1.832080200501253,
+      "grad_norm": 0.1294950395822525,
+      "learning_rate": 6.956804479234634e-05,
+      "loss": 0.044572627544403075,
+      "mean_token_accuracy": 0.9860877647995949,
+      "num_tokens": 2068358.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.0527445848332718,
+      "epoch": 1.8464017185821697,
+      "grad_norm": 0.13190579414367676,
+      "learning_rate": 6.810260736447529e-05,
+      "loss": 0.045303305983543395,
+      "mean_token_accuracy": 0.9857252709567547,
+      "num_tokens": 2084760.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.04997822809964418,
+      "epoch": 1.8607232366630861,
+      "grad_norm": 0.1623498499393463,
+      "learning_rate": 6.664475683491796e-05,
+      "loss": 0.04508442878723144,
+      "mean_token_accuracy": 0.9847477197647094,
+      "num_tokens": 2100672.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.04634254523552954,
+      "epoch": 1.8750447547440028,
+      "grad_norm": 0.12624591588974,
+      "learning_rate": 6.519483995815205e-05,
+      "loss": 0.04394125938415527,
+      "mean_token_accuracy": 0.9861477628350258,
+      "num_tokens": 2116931.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.04369507026858628,
+      "epoch": 1.8893662728249194,
+      "grad_norm": 0.12619736790657043,
+      "learning_rate": 6.375320160161031e-05,
+      "loss": 0.04355749487876892,
+      "mean_token_accuracy": 0.9859536297619342,
+      "num_tokens": 2133276.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.051831772667355835,
+      "epoch": 1.903687790905836,
+      "grad_norm": 0.1626484990119934,
+      "learning_rate": 6.232018466365247e-05,
+      "loss": 0.04528863728046417,
+      "mean_token_accuracy": 0.9840523652732373,
+      "num_tokens": 2149246.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.0481730573810637,
+      "epoch": 1.9180093089867527,
+      "grad_norm": 0.12361084669828415,
+      "learning_rate": 6.089612999200596e-05,
+      "loss": 0.04389986991882324,
+      "mean_token_accuracy": 0.9851875752210617,
+      "num_tokens": 2165381.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.044324265141040084,
+      "epoch": 1.9323308270676691,
+      "grad_norm": 0.11117231100797653,
+      "learning_rate": 5.948137630269406e-05,
+      "loss": 0.043296205997467044,
+      "mean_token_accuracy": 0.9862881116569042,
+      "num_tokens": 2181734.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.047920829802751544,
+      "epoch": 1.9466523451485858,
+      "grad_norm": 0.1605246663093567,
+      "learning_rate": 5.807626009947127e-05,
+      "loss": 0.043716180324554446,
+      "mean_token_accuracy": 0.9853886358439923,
+      "num_tokens": 2197884.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.04748345841653645,
+      "epoch": 1.9609738632295022,
+      "grad_norm": 0.14858777821063995,
+      "learning_rate": 5.668111559378471e-05,
+      "loss": 0.04364239573478699,
+      "mean_token_accuracy": 0.9847554847598076,
+      "num_tokens": 2214119.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.04570549342315644,
+      "epoch": 1.9752953813104188,
+      "grad_norm": 0.13550825417041779,
+      "learning_rate": 5.529627462528091e-05,
+      "loss": 0.04398110508918762,
+      "mean_token_accuracy": 0.9848537050187588,
+      "num_tokens": 2230494.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.04643255970440805,
+      "epoch": 1.9896168993913355,
+      "grad_norm": 0.11942434310913086,
+      "learning_rate": 5.392206658287669e-05,
+      "loss": 0.04292392134666443,
+      "mean_token_accuracy": 0.9843832887709141,
+      "num_tokens": 2247033.0,
+      "step": 1390
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.04502911090242619,
+      "eval_loss": 0.04353385418653488,
+      "eval_mean_token_accuracy": 0.9857475236565078,
+      "eval_num_tokens": 2258760.0,
+      "eval_runtime": 131.9942,
+      "eval_samples_per_second": 2.227,
+      "eval_steps_per_second": 2.227,
+      "step": 1398
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2097,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.334138813005824e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

lora_adapter/checkpoint-1398/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e3a9b47617458e67ed5dfbd51df40601253710b43c78072693271669fd135a
+size 5713

lora_adapter/checkpoint-2097/README.md

@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Llama-3.2-1B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.21.0

lora_adapter/checkpoint-2097/adapter_config.json

@@ -0,0 +1,51 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "kasa_config": null,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.21.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

lora_adapter/checkpoint-2097/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a37d910a4f15fea01038dd325891944e24ee48e481f8c85466c76d53a832616
+size 45118424

lora_adapter/checkpoint-2097/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

lora_adapter/checkpoint-2097/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81935cca33f0ff8403ceefc30634e0c2360935c2262e833143916cb69e3e23cb
+size 90359883

lora_adapter/checkpoint-2097/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:391d01d3aeb4a35151817d446e4ba0b9c8a04084ae1b1b66eda188a30729da0a
+size 14455

lora_adapter/checkpoint-2097/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41979c24a34738840d5007f1233bfc5e6d147fd90ac26425fccad8b6d6e6d4f0
+size 1465

lora_adapter/checkpoint-2097/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

lora_adapter/checkpoint-2097/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|finetune_right_pad_id|>",
+  "padding_side": "left",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null
+}

lora_adapter/checkpoint-2097/trainer_state.json

@@ -0,0 +1,2157 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 2097,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 3.418184721469879,
+      "epoch": 0.014321518080916577,
+      "grad_norm": 5.165404319763184,
+      "learning_rate": 3e-05,
+      "loss": 4.75037727355957,
+      "mean_token_accuracy": 0.3666964743286371,
+      "num_tokens": 16368.0,
+      "step": 10
+    },
+    {
+      "entropy": 3.8264630317687987,
+      "epoch": 0.028643036161833155,
+      "grad_norm": 2.789987802505493,
+      "learning_rate": 6.333333333333333e-05,
+      "loss": 3.6756973266601562,
+      "mean_token_accuracy": 0.4501046933233738,
+      "num_tokens": 32587.0,
+      "step": 20
+    },
+    {
+      "entropy": 3.0331100687384605,
+      "epoch": 0.04296455424274973,
+      "grad_norm": 2.0175697803497314,
+      "learning_rate": 9.666666666666667e-05,
+      "loss": 2.153616523742676,
+      "mean_token_accuracy": 0.6448496542870998,
+      "num_tokens": 48627.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.7019604802131654,
+      "epoch": 0.05728607232366631,
+      "grad_norm": 1.4288526773452759,
+      "learning_rate": 0.00013000000000000002,
+      "loss": 1.1227694511413575,
+      "mean_token_accuracy": 0.8201614201068879,
+      "num_tokens": 64784.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.8430624332278966,
+      "epoch": 0.07160759040458288,
+      "grad_norm": 0.9607819318771362,
+      "learning_rate": 0.00016333333333333334,
+      "loss": 0.5127146720886231,
+      "mean_token_accuracy": 0.9230180129408836,
+      "num_tokens": 80994.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.3905979799106717,
+      "epoch": 0.08592910848549946,
+      "grad_norm": 0.8198261260986328,
+      "learning_rate": 0.00019666666666666666,
+      "loss": 0.3345895767211914,
+      "mean_token_accuracy": 0.9489256188273429,
+      "num_tokens": 97095.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.3071663906797767,
+      "epoch": 0.10025062656641603,
+      "grad_norm": 2.082451343536377,
+      "learning_rate": 0.00019999036690733865,
+      "loss": 0.29979329109191893,
+      "mean_token_accuracy": 0.9523964017629624,
+      "num_tokens": 113170.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.24958498124033213,
+      "epoch": 0.11457214464733262,
+      "grad_norm": 1.3423985242843628,
+      "learning_rate": 0.00019995706971049468,
+      "loss": 0.19564504623413087,
+      "mean_token_accuracy": 0.9659054979681969,
+      "num_tokens": 129473.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.16129616871476174,
+      "epoch": 0.1288936627282492,
+      "grad_norm": 0.6217024922370911,
+      "learning_rate": 0.00019989999740035867,
+      "loss": 0.1112561583518982,
+      "mean_token_accuracy": 0.9752564527094364,
+      "num_tokens": 145693.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.1593495703302324,
+      "epoch": 0.14321518080916576,
+      "grad_norm": 0.9386972784996033,
+      "learning_rate": 0.00019981916355176472,
+      "loss": 0.124180269241333,
+      "mean_token_accuracy": 0.9724250495433807,
+      "num_tokens": 161885.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.11440805536694824,
+      "epoch": 0.15753669889008234,
+      "grad_norm": 0.49029725790023804,
+      "learning_rate": 0.00019971458739130598,
+      "loss": 0.08359839916229247,
+      "mean_token_accuracy": 0.9790669329464435,
+      "num_tokens": 178194.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.16797136263921858,
+      "epoch": 0.17185821697099893,
+      "grad_norm": 0.4058690071105957,
+      "learning_rate": 0.00019958629379276138,
+      "loss": 0.14625961780548097,
+      "mean_token_accuracy": 0.9697998225688934,
+      "num_tokens": 194281.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.2288114172872156,
+      "epoch": 0.1861797350519155,
+      "grad_norm": 0.43757033348083496,
+      "learning_rate": 0.00019943431327117957,
+      "loss": 0.1752084255218506,
+      "mean_token_accuracy": 0.9670783936977386,
+      "num_tokens": 210353.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.14037263644859194,
+      "epoch": 0.20050125313283207,
+      "grad_norm": 0.4667147696018219,
+      "learning_rate": 0.00019925868197562062,
+      "loss": 0.09937882423400879,
+      "mean_token_accuracy": 0.9777047835290432,
+      "num_tokens": 226307.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.13454449912533165,
+      "epoch": 0.21482277121374865,
+      "grad_norm": 0.5024069547653198,
+      "learning_rate": 0.00019905944168055798,
+      "loss": 0.10526319742202758,
+      "mean_token_accuracy": 0.9757608838379384,
+      "num_tokens": 242366.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.10749808787368238,
+      "epoch": 0.22914428929466524,
+      "grad_norm": 0.3371286392211914,
+      "learning_rate": 0.00019883663977594208,
+      "loss": 0.07259818315505981,
+      "mean_token_accuracy": 0.9807105578482151,
+      "num_tokens": 258533.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.14817415219731628,
+      "epoch": 0.24346580737558182,
+      "grad_norm": 0.6096305251121521,
+      "learning_rate": 0.00019859032925592874,
+      "loss": 0.1135907769203186,
+      "mean_token_accuracy": 0.9745507664978504,
+      "num_tokens": 274810.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.1133763374760747,
+      "epoch": 0.2577873254564984,
+      "grad_norm": 0.41122370958328247,
+      "learning_rate": 0.00019832056870627417,
+      "loss": 0.09238196611404419,
+      "mean_token_accuracy": 0.977765066921711,
+      "num_tokens": 290922.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.07603299776092172,
+      "epoch": 0.272108843537415,
+      "grad_norm": 0.2798531949520111,
+      "learning_rate": 0.00019802742229040025,
+      "loss": 0.06427340507507324,
+      "mean_token_accuracy": 0.9813245065510273,
+      "num_tokens": 307118.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.11896687960252166,
+      "epoch": 0.2864303616183315,
+      "grad_norm": 0.24208419024944305,
+      "learning_rate": 0.00019771095973413293,
+      "loss": 0.10308414697647095,
+      "mean_token_accuracy": 0.9760065227746964,
+      "num_tokens": 323416.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.09079958228394389,
+      "epoch": 0.3007518796992481,
+      "grad_norm": 0.4041318893432617,
+      "learning_rate": 0.00019737125630911784,
+      "loss": 0.06735156178474426,
+      "mean_token_accuracy": 0.9801696293056011,
+      "num_tokens": 339621.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.09143369558732957,
+      "epoch": 0.3150733977801647,
+      "grad_norm": 0.3946863114833832,
+      "learning_rate": 0.00019700839281491662,
+      "loss": 0.07498582005500794,
+      "mean_token_accuracy": 0.9808824859559536,
+      "num_tokens": 355782.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.12254208489321172,
+      "epoch": 0.32939491586108127,
+      "grad_norm": 0.31145262718200684,
+      "learning_rate": 0.00019662245555978846,
+      "loss": 0.09450015425682068,
+      "mean_token_accuracy": 0.976518415659666,
+      "num_tokens": 371844.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.10461308304220437,
+      "epoch": 0.34371643394199786,
+      "grad_norm": 0.3476834297180176,
+      "learning_rate": 0.00019621353634016153,
+      "loss": 0.08276034593582153,
+      "mean_token_accuracy": 0.9795444570481777,
+      "num_tokens": 388015.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.11855657077394426,
+      "epoch": 0.35803795202291444,
+      "grad_norm": 0.32865747809410095,
+      "learning_rate": 0.00019578173241879872,
+      "loss": 0.09439409971237182,
+      "mean_token_accuracy": 0.9776719771325588,
+      "num_tokens": 404196.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.11360202138312162,
+      "epoch": 0.372359470103831,
+      "grad_norm": 0.41493919491767883,
+      "learning_rate": 0.0001953271465016635,
+      "loss": 0.09676467776298522,
+      "mean_token_accuracy": 0.9767774596810341,
+      "num_tokens": 420199.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.1050779873970896,
+      "epoch": 0.3866809881847476,
+      "grad_norm": 0.22015489637851715,
+      "learning_rate": 0.000194849886713491,
+      "loss": 0.07251381278038024,
+      "mean_token_accuracy": 0.9796808525919914,
+      "num_tokens": 436468.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.10228129292372615,
+      "epoch": 0.40100250626566414,
+      "grad_norm": 0.27469080686569214,
+      "learning_rate": 0.00019435006657206997,
+      "loss": 0.09110605716705322,
+      "mean_token_accuracy": 0.9781862519681453,
+      "num_tokens": 452720.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.09573564094025641,
+      "epoch": 0.4153240243465807,
+      "grad_norm": 0.20515574514865875,
+      "learning_rate": 0.00019382780496124253,
+      "loss": 0.06511516571044922,
+      "mean_token_accuracy": 0.9820534713566303,
+      "num_tokens": 468938.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.06887364573776722,
+      "epoch": 0.4296455424274973,
+      "grad_norm": 0.2742563784122467,
+      "learning_rate": 0.00019328322610262703,
+      "loss": 0.053380900621414186,
+      "mean_token_accuracy": 0.9827778674662113,
+      "num_tokens": 485200.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.10576252338942141,
+      "epoch": 0.4439670605084139,
+      "grad_norm": 0.22944959998130798,
+      "learning_rate": 0.00019271645952607163,
+      "loss": 0.09060812592506409,
+      "mean_token_accuracy": 0.9780363194644451,
+      "num_tokens": 501474.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.1117024946026504,
+      "epoch": 0.4582885785893305,
+      "grad_norm": 0.2669987678527832,
+      "learning_rate": 0.0001921276400388451,
+      "loss": 0.07678983211517335,
+      "mean_token_accuracy": 0.9798587985336781,
+      "num_tokens": 517580.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.10053576473146678,
+      "epoch": 0.47261009667024706,
+      "grad_norm": 0.24858009815216064,
+      "learning_rate": 0.00019151690769357265,
+      "loss": 0.08534708023071289,
+      "mean_token_accuracy": 0.9798071250319481,
+      "num_tokens": 533788.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.13230610005557536,
+      "epoch": 0.48693161475116364,
+      "grad_norm": 0.2949056923389435,
+      "learning_rate": 0.00019088440775492384,
+      "loss": 0.09592426419258118,
+      "mean_token_accuracy": 0.9757725939154624,
+      "num_tokens": 549966.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.06407897693570704,
+      "epoch": 0.5012531328320802,
+      "grad_norm": 0.24437105655670166,
+      "learning_rate": 0.00019023029066506087,
+      "loss": 0.05607050061225891,
+      "mean_token_accuracy": 0.9830520637333393,
+      "num_tokens": 566066.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.06379525621887297,
+      "epoch": 0.5155746509129968,
+      "grad_norm": 0.2406875491142273,
+      "learning_rate": 0.00018955471200785551,
+      "loss": 0.05550529956817627,
+      "mean_token_accuracy": 0.9826574549078941,
+      "num_tokens": 582036.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.10768530536442995,
+      "epoch": 0.5298961689939133,
+      "grad_norm": 0.19389888644218445,
+      "learning_rate": 0.00018885783247188305,
+      "loss": 0.09576615095138549,
+      "mean_token_accuracy": 0.9779540851712227,
+      "num_tokens": 598182.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.12297249888069928,
+      "epoch": 0.54421768707483,
+      "grad_norm": 0.33548423647880554,
+      "learning_rate": 0.0001881398178122017,
+      "loss": 0.08422422409057617,
+      "mean_token_accuracy": 0.9791318334639072,
+      "num_tokens": 614468.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.06059104287996888,
+      "epoch": 0.5585392051557465,
+      "grad_norm": 0.34327495098114014,
+      "learning_rate": 0.0001874008388109276,
+      "loss": 0.05207563042640686,
+      "mean_token_accuracy": 0.9843599937856198,
+      "num_tokens": 630800.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.09479554321151226,
+      "epoch": 0.572860723236663,
+      "grad_norm": 0.3410899043083191,
+      "learning_rate": 0.00018664107123661337,
+      "loss": 0.0662381649017334,
+      "mean_token_accuracy": 0.9813465528190136,
+      "num_tokens": 646956.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.09436679913196713,
+      "epoch": 0.5871822413175797,
+      "grad_norm": 0.8792180418968201,
+      "learning_rate": 0.00018586069580244128,
+      "loss": 0.07904249429702759,
+      "mean_token_accuracy": 0.9798643849790096,
+      "num_tokens": 663240.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.09691147103440016,
+      "epoch": 0.6015037593984962,
+      "grad_norm": 0.23540320992469788,
+      "learning_rate": 0.00018505989812323993,
+      "loss": 0.06907619833946228,
+      "mean_token_accuracy": 0.9815059810876846,
+      "num_tokens": 679484.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.07770795098040253,
+      "epoch": 0.6158252774794128,
+      "grad_norm": 0.28902187943458557,
+      "learning_rate": 0.0001842388686713351,
+      "loss": 0.05818814635276794,
+      "mean_token_accuracy": 0.9836779788136483,
+      "num_tokens": 695720.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.06032483666203916,
+      "epoch": 0.6301467955603294,
+      "grad_norm": 0.328427791595459,
+      "learning_rate": 0.0001833978027312453,
+      "loss": 0.053039717674255374,
+      "mean_token_accuracy": 0.9840192586183548,
+      "num_tokens": 711945.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.0849465676350519,
+      "epoch": 0.644468313641246,
+      "grad_norm": 0.16869916021823883,
+      "learning_rate": 0.00018253690035323285,
+      "loss": 0.06781792044639587,
+      "mean_token_accuracy": 0.9806929357349873,
+      "num_tokens": 728029.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.08969353097490966,
+      "epoch": 0.6587898317221625,
+      "grad_norm": 0.18590566515922546,
+      "learning_rate": 0.0001816563663057211,
+      "loss": 0.06440567970275879,
+      "mean_token_accuracy": 0.9825256459414959,
+      "num_tokens": 744330.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.06217158674262464,
+      "epoch": 0.6731113498030791,
+      "grad_norm": 0.27869918942451477,
+      "learning_rate": 0.00018075641002658976,
+      "loss": 0.053045254945755,
+      "mean_token_accuracy": 0.9843784637749196,
+      "num_tokens": 760323.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.0674417833564803,
+      "epoch": 0.6874328678839957,
+      "grad_norm": 0.1830916553735733,
+      "learning_rate": 0.00017983724557335925,
+      "loss": 0.055755794048309326,
+      "mean_token_accuracy": 0.9838082902133465,
+      "num_tokens": 776628.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.0802982683526352,
+      "epoch": 0.7017543859649122,
+      "grad_norm": 0.4552997350692749,
+      "learning_rate": 0.00017889909157227653,
+      "loss": 0.060858166217803954,
+      "mean_token_accuracy": 0.9824069820344448,
+      "num_tokens": 792838.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.07723174102138727,
+      "epoch": 0.7160759040458289,
+      "grad_norm": 0.18987013399600983,
+      "learning_rate": 0.00017794217116631417,
+      "loss": 0.060531008243560794,
+      "mean_token_accuracy": 0.9814355976879596,
+      "num_tokens": 808999.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.05520166652277112,
+      "epoch": 0.7303974221267454,
+      "grad_norm": 0.18790723383426666,
+      "learning_rate": 0.00017696671196209496,
+      "loss": 0.05018657445907593,
+      "mean_token_accuracy": 0.983309705555439,
+      "num_tokens": 825062.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.05418659960851073,
+      "epoch": 0.744718940207662,
+      "grad_norm": 0.160777747631073,
+      "learning_rate": 0.00017597294597575507,
+      "loss": 0.04926420748233795,
+      "mean_token_accuracy": 0.9841075465083122,
+      "num_tokens": 841241.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.1152086689369753,
+      "epoch": 0.7590404582885786,
+      "grad_norm": 0.6535390615463257,
+      "learning_rate": 0.0001749611095777581,
+      "loss": 0.0968184232711792,
+      "mean_token_accuracy": 0.975805052369833,
+      "num_tokens": 857300.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.07658788892440498,
+      "epoch": 0.7733619763694952,
+      "grad_norm": 0.1728331297636032,
+      "learning_rate": 0.00017393144343667364,
+      "loss": 0.06037890911102295,
+      "mean_token_accuracy": 0.9823231592774391,
+      "num_tokens": 873434.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.09907218038570136,
+      "epoch": 0.7876834944504117,
+      "grad_norm": 0.22389887273311615,
+      "learning_rate": 0.00017288419246193373,
+      "loss": 0.07112311124801636,
+      "mean_token_accuracy": 0.9803691074252129,
+      "num_tokens": 889567.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.05675842456985265,
+      "epoch": 0.8020050125313283,
+      "grad_norm": 0.17525967955589294,
+      "learning_rate": 0.00017181960574557992,
+      "loss": 0.04961739480495453,
+      "mean_token_accuracy": 0.9838187299668789,
+      "num_tokens": 905625.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.08107102129142732,
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.17234288156032562,
+      "learning_rate": 0.00017073793650301624,
+      "loss": 0.06352313160896302,
+      "mean_token_accuracy": 0.9820462837815285,
+      "num_tokens": 921757.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.061689483653754,
+      "epoch": 0.8306480486931614,
+      "grad_norm": 0.23366186022758484,
+      "learning_rate": 0.00016963944201278097,
+      "loss": 0.051079404354095456,
+      "mean_token_accuracy": 0.982898436486721,
+      "num_tokens": 937984.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.06318069302942604,
+      "epoch": 0.8449695667740781,
+      "grad_norm": 0.19096462428569794,
+      "learning_rate": 0.00016852438355535214,
+      "loss": 0.05602290034294129,
+      "mean_token_accuracy": 0.9843950115144253,
+      "num_tokens": 954323.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.05533521031029522,
+      "epoch": 0.8592910848549946,
+      "grad_norm": 0.21601422131061554,
+      "learning_rate": 0.00016739302635100108,
+      "loss": 0.04836077392101288,
+      "mean_token_accuracy": 0.9856260627508163,
+      "num_tokens": 970505.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.05332150736358017,
+      "epoch": 0.8736126029359113,
+      "grad_norm": 0.16544337570667267,
+      "learning_rate": 0.00016624563949670904,
+      "loss": 0.04835851788520813,
+      "mean_token_accuracy": 0.9843276873230934,
+      "num_tokens": 986682.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.052138336235657334,
+      "epoch": 0.8879341210168278,
+      "grad_norm": 0.1802208125591278,
+      "learning_rate": 0.0001650824959021614,
+      "loss": 0.05234106183052063,
+      "mean_token_accuracy": 0.9847004011273384,
+      "num_tokens": 1002790.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.055844594305381176,
+      "epoch": 0.9022556390977443,
+      "grad_norm": 0.15213149785995483,
+      "learning_rate": 0.00016390387222483537,
+      "loss": 0.04984959363937378,
+      "mean_token_accuracy": 0.983958077430725,
+      "num_tokens": 1019089.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.06128789323847741,
+      "epoch": 0.916577157178661,
+      "grad_norm": 0.12318123877048492,
+      "learning_rate": 0.00016271004880419608,
+      "loss": 0.05124086737632751,
+      "mean_token_accuracy": 0.9843113601207734,
+      "num_tokens": 1035332.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.0652699463069439,
+      "epoch": 0.9308986752595775,
+      "grad_norm": 0.16328729689121246,
+      "learning_rate": 0.00016150130959501687,
+      "loss": 0.051763099431991574,
+      "mean_token_accuracy": 0.9837749816477299,
+      "num_tokens": 1051331.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.0749959686305374,
+      "epoch": 0.9452201933404941,
+      "grad_norm": 0.2559461295604706,
+      "learning_rate": 0.0001602779420998398,
+      "loss": 0.05571107268333435,
+      "mean_token_accuracy": 0.9834148533642292,
+      "num_tokens": 1067608.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.05268463878892362,
+      "epoch": 0.9595417114214106,
+      "grad_norm": 0.14703021943569183,
+      "learning_rate": 0.00015904023730059228,
+      "loss": 0.04806509017944336,
+      "mean_token_accuracy": 0.9848555572330951,
+      "num_tokens": 1083704.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.08527488259132951,
+      "epoch": 0.9738632295023273,
+      "grad_norm": 0.27216821908950806,
+      "learning_rate": 0.00015778848958937586,
+      "loss": 0.061145973205566403,
+      "mean_token_accuracy": 0.9831270687282085,
+      "num_tokens": 1099812.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.05362487006932497,
+      "epoch": 0.9881847475832438,
+      "grad_norm": 0.20429278910160065,
+      "learning_rate": 0.00015652299669844438,
+      "loss": 0.04693828225135803,
+      "mean_token_accuracy": 0.9854582466185093,
+      "num_tokens": 1116039.0,
+      "step": 690
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 0.05509232616267439,
+      "eval_loss": 0.04924684017896652,
+      "eval_mean_token_accuracy": 0.9852213520987504,
+      "eval_num_tokens": 1129380.0,
+      "eval_runtime": 132.3751,
+      "eval_samples_per_second": 2.221,
+      "eval_steps_per_second": 2.221,
+      "step": 699
+    },
+    {
+      "entropy": 0.06238776402598297,
+      "epoch": 1.0014321518080918,
+      "grad_norm": 0.12905091047286987,
+      "learning_rate": 0.00015524405962938714,
+      "loss": 0.05233152508735657,
+      "mean_token_accuracy": 0.9846261304778021,
+      "num_tokens": 1130996.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.058638660283759234,
+      "epoch": 1.0157536698890082,
+      "grad_norm": 0.2219594568014145,
+      "learning_rate": 0.00015395198258153498,
+      "loss": 0.05078426599502563,
+      "mean_token_accuracy": 0.9838196195662021,
+      "num_tokens": 1147116.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.05818793845828622,
+      "epoch": 1.0300751879699248,
+      "grad_norm": 0.6569804549217224,
+      "learning_rate": 0.00015264707287960521,
+      "loss": 0.05066579580307007,
+      "mean_token_accuracy": 0.9846140623092652,
+      "num_tokens": 1163282.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.054456926276907326,
+      "epoch": 1.0443967060508415,
+      "grad_norm": 0.19272315502166748,
+      "learning_rate": 0.00015132964090060357,
+      "loss": 0.048507091403007505,
+      "mean_token_accuracy": 0.9837053954601288,
+      "num_tokens": 1179237.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.04848019825294614,
+      "epoch": 1.0587182241317579,
+      "grad_norm": 0.1253335028886795,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 0.04691898822784424,
+      "mean_token_accuracy": 0.9845252685248852,
+      "num_tokens": 1195531.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.06931644980795681,
+      "epoch": 1.0730397422126745,
+      "grad_norm": 0.19210802018642426,
+      "learning_rate": 0.0001486584664371961,
+      "loss": 0.05346042513847351,
+      "mean_token_accuracy": 0.9840465135872364,
+      "num_tokens": 1212061.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.0602082708152011,
+      "epoch": 1.0873612602935911,
+      "grad_norm": 0.18707653880119324,
+      "learning_rate": 0.00014730535930030172,
+      "loss": 0.04904778599739075,
+      "mean_token_accuracy": 0.9847656920552254,
+      "num_tokens": 1228125.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.057827019784599545,
+      "epoch": 1.1016827783745078,
+      "grad_norm": 0.1590399444103241,
+      "learning_rate": 0.00014594100043023887,
+      "loss": 0.048907867074012755,
+      "mean_token_accuracy": 0.9835596241056919,
+      "num_tokens": 1244380.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.05155087348539382,
+      "epoch": 1.1160042964554242,
+      "grad_norm": 0.13960455358028412,
+      "learning_rate": 0.00014456571434419074,
+      "loss": 0.04657362103462219,
+      "mean_token_accuracy": 0.9841499179601669,
+      "num_tokens": 1260560.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.0582313765771687,
+      "epoch": 1.1303258145363408,
+      "grad_norm": 0.1564759463071823,
+      "learning_rate": 0.0001431798281584144,
+      "loss": 0.051281589269638064,
+      "mean_token_accuracy": 0.9844076797366142,
+      "num_tokens": 1276932.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.051610140060074625,
+      "epoch": 1.1446473326172575,
+      "grad_norm": 0.16707740724086761,
+      "learning_rate": 0.0001417836715104349,
+      "loss": 0.04861922562122345,
+      "mean_token_accuracy": 0.9845031477510929,
+      "num_tokens": 1293128.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.05285086496733129,
+      "epoch": 1.158968850698174,
+      "grad_norm": 0.13717925548553467,
+      "learning_rate": 0.00014037757648064018,
+      "loss": 0.04775188565254211,
+      "mean_token_accuracy": 0.9854285247623921,
+      "num_tokens": 1309523.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.05543681618291885,
+      "epoch": 1.1732903687790905,
+      "grad_norm": 0.15092015266418457,
+      "learning_rate": 0.00013896187751329437,
+      "loss": 0.046713563799858096,
+      "mean_token_accuracy": 0.9847700297832489,
+      "num_tokens": 1325813.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.04951737595256418,
+      "epoch": 1.1876118868600072,
+      "grad_norm": 0.1319475620985031,
+      "learning_rate": 0.00013753691133698944,
+      "loss": 0.045718804001808167,
+      "mean_token_accuracy": 0.9848351627588272,
+      "num_tokens": 1342049.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.05453478880226612,
+      "epoch": 1.2019334049409238,
+      "grad_norm": 0.1564560830593109,
+      "learning_rate": 0.000136103016884553,
+      "loss": 0.048373016715049746,
+      "mean_token_accuracy": 0.9833561331033707,
+      "num_tokens": 1358086.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.05261064667720348,
+      "epoch": 1.2162549230218402,
+      "grad_norm": 0.1645234227180481,
+      "learning_rate": 0.00013466053521243214,
+      "loss": 0.047138947248458865,
+      "mean_token_accuracy": 0.9853625297546387,
+      "num_tokens": 1373958.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.04612366235814989,
+      "epoch": 1.2305764411027569,
+      "grad_norm": 0.13810141384601593,
+      "learning_rate": 0.000133209809419572,
+      "loss": 0.04452601969242096,
+      "mean_token_accuracy": 0.9859118059277534,
+      "num_tokens": 1390120.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.048122103605419395,
+      "epoch": 1.2448979591836735,
+      "grad_norm": 0.16478563845157623,
+      "learning_rate": 0.00013175118456580857,
+      "loss": 0.04704745411872864,
+      "mean_token_accuracy": 0.9848096296191216,
+      "num_tokens": 1406408.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.05111220530234277,
+      "epoch": 1.25921947726459,
+      "grad_norm": 0.14047689735889435,
+      "learning_rate": 0.00013028500758979506,
+      "loss": 0.04449088275432587,
+      "mean_token_accuracy": 0.9855061650276185,
+      "num_tokens": 1422673.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.050457812706008555,
+      "epoch": 1.2735409953455066,
+      "grad_norm": 0.14891056716442108,
+      "learning_rate": 0.0001288116272264816,
+      "loss": 0.0456391453742981,
+      "mean_token_accuracy": 0.9854636572301387,
+      "num_tokens": 1438953.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.04946525213308632,
+      "epoch": 1.2878625134264232,
+      "grad_norm": 0.16796022653579712,
+      "learning_rate": 0.00012733139392416722,
+      "loss": 0.045487576723098756,
+      "mean_token_accuracy": 0.9840421840548516,
+      "num_tokens": 1455012.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.049669814435765146,
+      "epoch": 1.3021840315073399,
+      "grad_norm": 0.15355756878852844,
+      "learning_rate": 0.0001258446597611447,
+      "loss": 0.04645189344882965,
+      "mean_token_accuracy": 0.9850803188979625,
+      "num_tokens": 1471054.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.048828864190727475,
+      "epoch": 1.3165055495882563,
+      "grad_norm": 0.1641317456960678,
+      "learning_rate": 0.00012435177836195776,
+      "loss": 0.04594441056251526,
+      "mean_token_accuracy": 0.9857151001691818,
+      "num_tokens": 1487139.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.04583641842473298,
+      "epoch": 1.330827067669173,
+      "grad_norm": 0.15289618074893951,
+      "learning_rate": 0.0001228531048132901,
+      "loss": 0.04497186541557312,
+      "mean_token_accuracy": 0.9860778227448463,
+      "num_tokens": 1503603.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.0469824765343219,
+      "epoch": 1.3451485857500896,
+      "grad_norm": 0.13089172542095184,
+      "learning_rate": 0.00012134899557950698,
+      "loss": 0.04497265219688416,
+      "mean_token_accuracy": 0.9844655573368073,
+      "num_tokens": 1519909.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.05958397570066154,
+      "epoch": 1.359470103831006,
+      "grad_norm": 0.3665521442890167,
+      "learning_rate": 0.000119839808417869,
+      "loss": 0.04814833104610443,
+      "mean_token_accuracy": 0.9847050316631794,
+      "num_tokens": 1536192.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.05206554315518588,
+      "epoch": 1.3737916219119226,
+      "grad_norm": 0.15439002215862274,
+      "learning_rate": 0.00011832590229343835,
+      "loss": 0.045511645078659055,
+      "mean_token_accuracy": 0.9845592580735684,
+      "num_tokens": 1552241.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.05069404630921781,
+      "epoch": 1.3881131399928393,
+      "grad_norm": 0.1370713859796524,
+      "learning_rate": 0.00011680763729369783,
+      "loss": 0.04533308446407318,
+      "mean_token_accuracy": 0.9854350961744786,
+      "num_tokens": 1568661.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.048878395347855985,
+      "epoch": 1.402434658073756,
+      "grad_norm": 0.15589790046215057,
+      "learning_rate": 0.00011528537454290294,
+      "loss": 0.046069884300231935,
+      "mean_token_accuracy": 0.9837729744613171,
+      "num_tokens": 1585086.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.045904085785150525,
+      "epoch": 1.4167561761546725,
+      "grad_norm": 0.14884407818317413,
+      "learning_rate": 0.00011375947611618716,
+      "loss": 0.04572470188140869,
+      "mean_token_accuracy": 0.9846699081361294,
+      "num_tokens": 1601155.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.056870630197227004,
+      "epoch": 1.431077694235589,
+      "grad_norm": 0.21461369097232819,
+      "learning_rate": 0.00011223030495344127,
+      "loss": 0.05038630962371826,
+      "mean_token_accuracy": 0.9850619569420814,
+      "num_tokens": 1617145.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.04888233456294984,
+      "epoch": 1.4453992123165056,
+      "grad_norm": 0.13056223094463348,
+      "learning_rate": 0.00011069822477298706,
+      "loss": 0.0462115079164505,
+      "mean_token_accuracy": 0.9843272507190705,
+      "num_tokens": 1633412.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.05794957473408431,
+      "epoch": 1.459720730397422,
+      "grad_norm": 0.3261290490627289,
+      "learning_rate": 0.0001091635999850655,
+      "loss": 0.04766329824924469,
+      "mean_token_accuracy": 0.9835515014827252,
+      "num_tokens": 1649727.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.048225057567469774,
+      "epoch": 1.4740422484783386,
+      "grad_norm": 0.130174919962883,
+      "learning_rate": 0.00010762679560516067,
+      "loss": 0.04580983817577362,
+      "mean_token_accuracy": 0.9854344628751278,
+      "num_tokens": 1665979.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.061461174837313594,
+      "epoch": 1.4883637665592553,
+      "grad_norm": 0.1320205181837082,
+      "learning_rate": 0.00010608817716717976,
+      "loss": 0.048619455099105834,
+      "mean_token_accuracy": 0.9839078761637211,
+      "num_tokens": 1682011.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.052603658544830977,
+      "epoch": 1.502685284640172,
+      "grad_norm": 0.16996844112873077,
+      "learning_rate": 0.00010454811063650957,
+      "loss": 0.04834533929824829,
+      "mean_token_accuracy": 0.9843780644237995,
+      "num_tokens": 1697914.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.054306549951434135,
+      "epoch": 1.5170068027210886,
+      "grad_norm": 0.1440720409154892,
+      "learning_rate": 0.0001030069623229704,
+      "loss": 0.04781841635704041,
+      "mean_token_accuracy": 0.9854689165949821,
+      "num_tokens": 1714070.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.05636612819507718,
+      "epoch": 1.531328320802005,
+      "grad_norm": 0.14176875352859497,
+      "learning_rate": 0.00010146509879368808,
+      "loss": 0.04686476588249207,
+      "mean_token_accuracy": 0.9847516976296902,
+      "num_tokens": 1730131.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.05093875625170767,
+      "epoch": 1.5456498388829216,
+      "grad_norm": 0.1596251130104065,
+      "learning_rate": 9.99228867859049e-05,
+      "loss": 0.04491396546363831,
+      "mean_token_accuracy": 0.9852274671196938,
+      "num_tokens": 1746342.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.04888144808355719,
+      "epoch": 1.559971356963838,
+      "grad_norm": 0.20582543313503265,
+      "learning_rate": 9.838069311974986e-05,
+      "loss": 0.0455554723739624,
+      "mean_token_accuracy": 0.9859486259520054,
+      "num_tokens": 1762591.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.05331454309634864,
+      "epoch": 1.5742928750447547,
+      "grad_norm": 0.1273772269487381,
+      "learning_rate": 9.683888461098933e-05,
+      "loss": 0.04549239873886109,
+      "mean_token_accuracy": 0.9847855247557163,
+      "num_tokens": 1778625.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.048709102207794785,
+      "epoch": 1.5886143931256713,
+      "grad_norm": 0.11710993200540543,
+      "learning_rate": 9.529782798377877e-05,
+      "loss": 0.047628313302993774,
+      "mean_token_accuracy": 0.9854380682110786,
+      "num_tokens": 1794731.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.04785530364606529,
+      "epoch": 1.602935911206588,
+      "grad_norm": 0.1247224509716034,
+      "learning_rate": 9.375788978343608e-05,
+      "loss": 0.04518147110939026,
+      "mean_token_accuracy": 0.9852628707885742,
+      "num_tokens": 1810816.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.05034571662545204,
+      "epoch": 1.6172574292875046,
+      "grad_norm": 0.14023028314113617,
+      "learning_rate": 9.221943628925789e-05,
+      "loss": 0.04668688774108887,
+      "mean_token_accuracy": 0.9848854929208756,
+      "num_tokens": 1826931.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.051596777117811146,
+      "epoch": 1.631578947368421,
+      "grad_norm": 0.1819605678319931,
+      "learning_rate": 9.068283342739853e-05,
+      "loss": 0.04736668169498444,
+      "mean_token_accuracy": 0.9847092233598232,
+      "num_tokens": 1842987.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.052112107467837634,
+      "epoch": 1.6459004654493377,
+      "grad_norm": 0.16193711757659912,
+      "learning_rate": 8.914844668383339e-05,
+      "loss": 0.046776899695396425,
+      "mean_token_accuracy": 0.9839866004884243,
+      "num_tokens": 1858881.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.04619899494573474,
+      "epoch": 1.660221983530254,
+      "grad_norm": 0.16279025375843048,
+      "learning_rate": 8.7616641017427e-05,
+      "loss": 0.04623554050922394,
+      "mean_token_accuracy": 0.9846173226833344,
+      "num_tokens": 1874895.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.047261204943060874,
+      "epoch": 1.6745435016111707,
+      "grad_norm": 0.1297980099916458,
+      "learning_rate": 8.608778077312633e-05,
+      "loss": 0.0437994509935379,
+      "mean_token_accuracy": 0.9848140060901642,
+      "num_tokens": 1891143.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.05167301250621677,
+      "epoch": 1.6888650196920874,
+      "grad_norm": 0.15251320600509644,
+      "learning_rate": 8.456222959530008e-05,
+      "loss": 0.045985713601112366,
+      "mean_token_accuracy": 0.9834124945104122,
+      "num_tokens": 1907142.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.04752049937378615,
+      "epoch": 1.703186537773004,
+      "grad_norm": 0.18589235842227936,
+      "learning_rate": 8.304035034124484e-05,
+      "loss": 0.04648590981960297,
+      "mean_token_accuracy": 0.9851418137550354,
+      "num_tokens": 1923277.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.048138466640375556,
+      "epoch": 1.7175080558539206,
+      "grad_norm": 0.16602511703968048,
+      "learning_rate": 8.152250499487816e-05,
+      "loss": 0.04516423046588898,
+      "mean_token_accuracy": 0.9846760742366314,
+      "num_tokens": 1939220.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.05131866903975606,
+      "epoch": 1.731829573934837,
+      "grad_norm": 0.20442146062850952,
+      "learning_rate": 8.000905458063957e-05,
+      "loss": 0.045997625589370726,
+      "mean_token_accuracy": 0.9854269422590732,
+      "num_tokens": 1955362.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.050703935651108624,
+      "epoch": 1.7461510920157537,
+      "grad_norm": 0.1469806581735611,
+      "learning_rate": 7.850035907761973e-05,
+      "loss": 0.045433256030082705,
+      "mean_token_accuracy": 0.9851394325494767,
+      "num_tokens": 1971476.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.04811225512530655,
+      "epoch": 1.76047261009667,
+      "grad_norm": 0.12229132652282715,
+      "learning_rate": 7.699677733393826e-05,
+      "loss": 0.04342373609542847,
+      "mean_token_accuracy": 0.9854144871234893,
+      "num_tokens": 1987751.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.046569342049770056,
+      "epoch": 1.7747941281775867,
+      "grad_norm": 0.14311222732067108,
+      "learning_rate": 7.549866698139048e-05,
+      "loss": 0.04468758106231689,
+      "mean_token_accuracy": 0.9856827527284622,
+      "num_tokens": 2003903.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.04997451878152788,
+      "epoch": 1.7891156462585034,
+      "grad_norm": 0.1998521387577057,
+      "learning_rate": 7.400638435038371e-05,
+      "loss": 0.04490821659564972,
+      "mean_token_accuracy": 0.9860882505774498,
+      "num_tokens": 2020064.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.047584796766750516,
+      "epoch": 1.80343716433942,
+      "grad_norm": 0.1390482783317566,
+      "learning_rate": 7.252028438518266e-05,
+      "loss": 0.044057485461235044,
+      "mean_token_accuracy": 0.9860201634466648,
+      "num_tokens": 2036074.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.046523316064849496,
+      "epoch": 1.8177586824203367,
+      "grad_norm": 0.15581190586090088,
+      "learning_rate": 7.1040720559485e-05,
+      "loss": 0.044152289628982544,
+      "mean_token_accuracy": 0.984954721480608,
+      "num_tokens": 2052410.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.048715499113313854,
+      "epoch": 1.832080200501253,
+      "grad_norm": 0.1294950395822525,
+      "learning_rate": 6.956804479234634e-05,
+      "loss": 0.044572627544403075,
+      "mean_token_accuracy": 0.9860877647995949,
+      "num_tokens": 2068358.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.0527445848332718,
+      "epoch": 1.8464017185821697,
+      "grad_norm": 0.13190579414367676,
+      "learning_rate": 6.810260736447529e-05,
+      "loss": 0.045303305983543395,
+      "mean_token_accuracy": 0.9857252709567547,
+      "num_tokens": 2084760.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.04997822809964418,
+      "epoch": 1.8607232366630861,
+      "grad_norm": 0.1623498499393463,
+      "learning_rate": 6.664475683491796e-05,
+      "loss": 0.04508442878723144,
+      "mean_token_accuracy": 0.9847477197647094,
+      "num_tokens": 2100672.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.04634254523552954,
+      "epoch": 1.8750447547440028,
+      "grad_norm": 0.12624591588974,
+      "learning_rate": 6.519483995815205e-05,
+      "loss": 0.04394125938415527,
+      "mean_token_accuracy": 0.9861477628350258,
+      "num_tokens": 2116931.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.04369507026858628,
+      "epoch": 1.8893662728249194,
+      "grad_norm": 0.12619736790657043,
+      "learning_rate": 6.375320160161031e-05,
+      "loss": 0.04355749487876892,
+      "mean_token_accuracy": 0.9859536297619342,
+      "num_tokens": 2133276.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.051831772667355835,
+      "epoch": 1.903687790905836,
+      "grad_norm": 0.1626484990119934,
+      "learning_rate": 6.232018466365247e-05,
+      "loss": 0.04528863728046417,
+      "mean_token_accuracy": 0.9840523652732373,
+      "num_tokens": 2149246.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.0481730573810637,
+      "epoch": 1.9180093089867527,
+      "grad_norm": 0.12361084669828415,
+      "learning_rate": 6.089612999200596e-05,
+      "loss": 0.04389986991882324,
+      "mean_token_accuracy": 0.9851875752210617,
+      "num_tokens": 2165381.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.044324265141040084,
+      "epoch": 1.9323308270676691,
+      "grad_norm": 0.11117231100797653,
+      "learning_rate": 5.948137630269406e-05,
+      "loss": 0.043296205997467044,
+      "mean_token_accuracy": 0.9862881116569042,
+      "num_tokens": 2181734.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.047920829802751544,
+      "epoch": 1.9466523451485858,
+      "grad_norm": 0.1605246663093567,
+      "learning_rate": 5.807626009947127e-05,
+      "loss": 0.043716180324554446,
+      "mean_token_accuracy": 0.9853886358439923,
+      "num_tokens": 2197884.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.04748345841653645,
+      "epoch": 1.9609738632295022,
+      "grad_norm": 0.14858777821063995,
+      "learning_rate": 5.668111559378471e-05,
+      "loss": 0.04364239573478699,
+      "mean_token_accuracy": 0.9847554847598076,
+      "num_tokens": 2214119.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.04570549342315644,
+      "epoch": 1.9752953813104188,
+      "grad_norm": 0.13550825417041779,
+      "learning_rate": 5.529627462528091e-05,
+      "loss": 0.04398110508918762,
+      "mean_token_accuracy": 0.9848537050187588,
+      "num_tokens": 2230494.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.04643255970440805,
+      "epoch": 1.9896168993913355,
+      "grad_norm": 0.11942434310913086,
+      "learning_rate": 5.392206658287669e-05,
+      "loss": 0.04292392134666443,
+      "mean_token_accuracy": 0.9843832887709141,
+      "num_tokens": 2247033.0,
+      "step": 1390
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.04502911090242619,
+      "eval_loss": 0.04353385418653488,
+      "eval_mean_token_accuracy": 0.9857475236565078,
+      "eval_num_tokens": 2258760.0,
+      "eval_runtime": 131.9942,
+      "eval_samples_per_second": 2.227,
+      "eval_steps_per_second": 2.227,
+      "step": 1398
+    },
+    {
+      "entropy": 0.04683936314304938,
+      "epoch": 2.0028643036161835,
+      "grad_norm": 0.13921518623828888,
+      "learning_rate": 5.255881832641285e-05,
+      "loss": 0.04282083213329315,
+      "mean_token_accuracy": 0.9850272552387135,
+      "num_tokens": 2262090.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.046890793996863064,
+      "epoch": 2.0171858216970997,
+      "grad_norm": 0.13835492730140686,
+      "learning_rate": 5.12068541089096e-05,
+      "loss": 0.044261837005615236,
+      "mean_token_accuracy": 0.9853827930986881,
+      "num_tokens": 2278199.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.04900680233258754,
+      "epoch": 2.0315073397780163,
+      "grad_norm": 0.15510393679141998,
+      "learning_rate": 4.9866495499441824e-05,
+      "loss": 0.04339735209941864,
+      "mean_token_accuracy": 0.986194907873869,
+      "num_tokens": 2294299.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.04782760886009783,
+      "epoch": 2.045828857858933,
+      "grad_norm": 0.1793293058872223,
+      "learning_rate": 4.853806130665289e-05,
+      "loss": 0.04327985346317291,
+      "mean_token_accuracy": 0.9857033453881741,
+      "num_tokens": 2310650.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.04814982507377863,
+      "epoch": 2.0601503759398496,
+      "grad_norm": 0.14288994669914246,
+      "learning_rate": 4.722186750292511e-05,
+      "loss": 0.04325262010097504,
+      "mean_token_accuracy": 0.9849577777087688,
+      "num_tokens": 2326808.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.047272500093095006,
+      "epoch": 2.0744718940207663,
+      "grad_norm": 0.1487351357936859,
+      "learning_rate": 4.5918227149224394e-05,
+      "loss": 0.04342210292816162,
+      "mean_token_accuracy": 0.9858958683907986,
+      "num_tokens": 2342947.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.04581530508585274,
+      "epoch": 2.088793412101683,
+      "grad_norm": 0.14583143591880798,
+      "learning_rate": 4.462745032063784e-05,
+      "loss": 0.042257273197174074,
+      "mean_token_accuracy": 0.9854277707636356,
+      "num_tokens": 2359246.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.0448954316554591,
+      "epoch": 2.1031149301825995,
+      "grad_norm": 0.1211216002702713,
+      "learning_rate": 4.334984403262146e-05,
+      "loss": 0.042856425046920776,
+      "mean_token_accuracy": 0.9853748582303524,
+      "num_tokens": 2375620.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.04505488155409694,
+      "epoch": 2.1174364482635157,
+      "grad_norm": 0.14479714632034302,
+      "learning_rate": 4.208571216797539e-05,
+      "loss": 0.043560296297073364,
+      "mean_token_accuracy": 0.9866597428917885,
+      "num_tokens": 2391656.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.046725850482471284,
+      "epoch": 2.1317579663444324,
+      "grad_norm": 0.12594418227672577,
+      "learning_rate": 4.0835355404564514e-05,
+      "loss": 0.04261627197265625,
+      "mean_token_accuracy": 0.9853146940469741,
+      "num_tokens": 2408129.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.04732067086733878,
+      "epoch": 2.146079484425349,
+      "grad_norm": 0.14261198043823242,
+      "learning_rate": 3.9599071143801135e-05,
+      "loss": 0.043105340003967284,
+      "mean_token_accuracy": 0.9855934135615826,
+      "num_tokens": 2424152.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.04741294668056071,
+      "epoch": 2.1604010025062657,
+      "grad_norm": 0.1186954528093338,
+      "learning_rate": 3.8377153439907266e-05,
+      "loss": 0.04385945200920105,
+      "mean_token_accuracy": 0.984919972717762,
+      "num_tokens": 2440315.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.04350291786249727,
+      "epoch": 2.1747225205871823,
+      "grad_norm": 0.12911053001880646,
+      "learning_rate": 3.716989292997264e-05,
+      "loss": 0.04217359125614166,
+      "mean_token_accuracy": 0.9859704896807671,
+      "num_tokens": 2456492.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.047570334770716725,
+      "epoch": 2.189044038668099,
+      "grad_norm": 0.14641720056533813,
+      "learning_rate": 3.597757676482591e-05,
+      "loss": 0.04339935481548309,
+      "mean_token_accuracy": 0.9846776388585567,
+      "num_tokens": 2472497.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.04768898223992437,
+      "epoch": 2.2033655567490156,
+      "grad_norm": 0.13052751123905182,
+      "learning_rate": 3.480048854073484e-05,
+      "loss": 0.043949815630912784,
+      "mean_token_accuracy": 0.984875550866127,
+      "num_tokens": 2488558.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.04821568271145225,
+      "epoch": 2.2176870748299318,
+      "grad_norm": 0.11001984775066376,
+      "learning_rate": 3.363890823195206e-05,
+      "loss": 0.04300495088100433,
+      "mean_token_accuracy": 0.984880269318819,
+      "num_tokens": 2504861.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.04534621348138899,
+      "epoch": 2.2320085929108484,
+      "grad_norm": 0.15328188240528107,
+      "learning_rate": 3.249311212412206e-05,
+      "loss": 0.04239692091941834,
+      "mean_token_accuracy": 0.9852856807410717,
+      "num_tokens": 2521276.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.04719252656213939,
+      "epoch": 2.246330110991765,
+      "grad_norm": 0.14670529961585999,
+      "learning_rate": 3.1363372748566176e-05,
+      "loss": 0.04274523854255676,
+      "mean_token_accuracy": 0.9852247647941113,
+      "num_tokens": 2537456.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.04555917175021022,
+      "epoch": 2.2606516290726817,
+      "grad_norm": 0.1444850116968155,
+      "learning_rate": 3.0249958817459722e-05,
+      "loss": 0.043156713247299194,
+      "mean_token_accuracy": 0.9854748398065567,
+      "num_tokens": 2553667.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.04679012491833419,
+      "epoch": 2.2749731471535983,
+      "grad_norm": 0.15165963768959045,
+      "learning_rate": 2.9153135159918143e-05,
+      "loss": 0.043199223279953,
+      "mean_token_accuracy": 0.9855281673371792,
+      "num_tokens": 2569644.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.05232465125154704,
+      "epoch": 2.289294665234515,
+      "grad_norm": 0.15277273952960968,
+      "learning_rate": 2.807316265900649e-05,
+      "loss": 0.0448356956243515,
+      "mean_token_accuracy": 0.9847052752971649,
+      "num_tokens": 2585486.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.04544073750730604,
+      "epoch": 2.3036161833154316,
+      "grad_norm": 0.14555519819259644,
+      "learning_rate": 2.7010298189687532e-05,
+      "loss": 0.04302690923213959,
+      "mean_token_accuracy": 0.9854784525930882,
+      "num_tokens": 2601595.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.04607099741697311,
+      "epoch": 2.317937701396348,
+      "grad_norm": 0.1324913501739502,
+      "learning_rate": 2.596479455772325e-05,
+      "loss": 0.041962969303131106,
+      "mean_token_accuracy": 0.9858310744166374,
+      "num_tokens": 2617821.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.044995435047894716,
+      "epoch": 2.3322592194772644,
+      "grad_norm": 0.16306857764720917,
+      "learning_rate": 2.4936900439544107e-05,
+      "loss": 0.043055152893066405,
+      "mean_token_accuracy": 0.9848604544997215,
+      "num_tokens": 2633863.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.04732508307788521,
+      "epoch": 2.346580737558181,
+      "grad_norm": 0.1405004858970642,
+      "learning_rate": 2.392686032310064e-05,
+      "loss": 0.04337474405765533,
+      "mean_token_accuracy": 0.9864315807819366,
+      "num_tokens": 2649678.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.04380548046901822,
+      "epoch": 2.3609022556390977,
+      "grad_norm": 0.17532312870025635,
+      "learning_rate": 2.2934914449711087e-05,
+      "loss": 0.041680130362510684,
+      "mean_token_accuracy": 0.9865069292485714,
+      "num_tokens": 2665870.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.045538244373165074,
+      "epoch": 2.3752237737200144,
+      "grad_norm": 0.11877548694610596,
+      "learning_rate": 2.1961298756919335e-05,
+      "loss": 0.04254555106163025,
+      "mean_token_accuracy": 0.9853885978460312,
+      "num_tokens": 2682257.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.048017595591954884,
+      "epoch": 2.389545291800931,
+      "grad_norm": 0.17336377501487732,
+      "learning_rate": 2.1006244822376332e-05,
+      "loss": 0.04367460310459137,
+      "mean_token_accuracy": 0.9859470762312412,
+      "num_tokens": 2698215.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.04923853036016226,
+      "epoch": 2.4038668098818476,
+      "grad_norm": 0.13735970854759216,
+      "learning_rate": 2.0069979808758586e-05,
+      "loss": 0.042500972747802734,
+      "mean_token_accuracy": 0.9848999679088593,
+      "num_tokens": 2714446.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.04572770516388118,
+      "epoch": 2.4181883279627643,
+      "grad_norm": 0.12805375456809998,
+      "learning_rate": 1.915272640973683e-05,
+      "loss": 0.042926487326622007,
+      "mean_token_accuracy": 0.9856123149394989,
+      "num_tokens": 2730316.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.04579656107816845,
+      "epoch": 2.4325098460436805,
+      "grad_norm": 0.17534421384334564,
+      "learning_rate": 1.8254702797007685e-05,
+      "loss": 0.042541873455047605,
+      "mean_token_accuracy": 0.9851514488458634,
+      "num_tokens": 2746546.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.044594893767498436,
+      "epoch": 2.446831364124597,
+      "grad_norm": 0.1623026430606842,
+      "learning_rate": 1.7376122568400522e-05,
+      "loss": 0.04285149872303009,
+      "mean_token_accuracy": 0.9852959342300892,
+      "num_tokens": 2762723.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.046070940676145256,
+      "epoch": 2.4611528822055138,
+      "grad_norm": 0.14286647737026215,
+      "learning_rate": 1.65171946970729e-05,
+      "loss": 0.04270348846912384,
+      "mean_token_accuracy": 0.9860672034323216,
+      "num_tokens": 2779253.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.04623219138011336,
+      "epoch": 2.4754744002864304,
+      "grad_norm": 0.13059432804584503,
+      "learning_rate": 1.5678123481805352e-05,
+      "loss": 0.042553290724754333,
+      "mean_token_accuracy": 0.9850045546889306,
+      "num_tokens": 2795498.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.04578286891337484,
+      "epoch": 2.489795918367347,
+      "grad_norm": 0.13886156678199768,
+      "learning_rate": 1.4859108498408403e-05,
+      "loss": 0.04241152107715607,
+      "mean_token_accuracy": 0.9858711421489715,
+      "num_tokens": 2811491.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.050244259042665364,
+      "epoch": 2.5041174364482637,
+      "grad_norm": 0.15488147735595703,
+      "learning_rate": 1.4060344552252702e-05,
+      "loss": 0.042510893940925595,
+      "mean_token_accuracy": 0.9862028263509274,
+      "num_tokens": 2827572.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.0463644573232159,
+      "epoch": 2.51843895452918,
+      "grad_norm": 0.17233335971832275,
+      "learning_rate": 1.3282021631934106e-05,
+      "loss": 0.042552676796913144,
+      "mean_token_accuracy": 0.9854075901210309,
+      "num_tokens": 2843974.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.04752129716798663,
+      "epoch": 2.5327604726100965,
+      "grad_norm": 0.1504816710948944,
+      "learning_rate": 1.2524324864084058e-05,
+      "loss": 0.04323456883430481,
+      "mean_token_accuracy": 0.9858408093452453,
+      "num_tokens": 2860032.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.04596719217952341,
+      "epoch": 2.547081990691013,
+      "grad_norm": 0.1210273876786232,
+      "learning_rate": 1.1787434469336766e-05,
+      "loss": 0.04099370539188385,
+      "mean_token_accuracy": 0.9863001741468906,
+      "num_tokens": 2876211.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.048594068735837936,
+      "epoch": 2.56140350877193,
+      "grad_norm": 0.1528884321451187,
+      "learning_rate": 1.1071525719463095e-05,
+      "loss": 0.04276636838912964,
+      "mean_token_accuracy": 0.9845349103212356,
+      "num_tokens": 2892183.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.04557484819088131,
+      "epoch": 2.5757250268528464,
+      "grad_norm": 0.15690766274929047,
+      "learning_rate": 1.0376768895681654e-05,
+      "loss": 0.042804256081581116,
+      "mean_token_accuracy": 0.9859333589673043,
+      "num_tokens": 2908651.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.0443979425355792,
+      "epoch": 2.590046544933763,
+      "grad_norm": 0.144444078207016,
+      "learning_rate": 9.703329248156711e-06,
+      "loss": 0.04173080623149872,
+      "mean_token_accuracy": 0.9864362731575966,
+      "num_tokens": 2924695.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.04709135969169438,
+      "epoch": 2.6043680630146797,
+      "grad_norm": 0.13330085575580597,
+      "learning_rate": 9.051366956693097e-06,
+      "loss": 0.042562901973724365,
+      "mean_token_accuracy": 0.9862306192517281,
+      "num_tokens": 2940873.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.046017447393387555,
+      "epoch": 2.6186895810955964,
+      "grad_norm": 0.14055359363555908,
+      "learning_rate": 8.42103709263682e-06,
+      "loss": 0.041986921429634096,
+      "mean_token_accuracy": 0.9856328696012497,
+      "num_tokens": 2957018.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.04738146187737584,
+      "epoch": 2.6330110991765125,
+      "grad_norm": 0.15382255613803864,
+      "learning_rate": 7.812489581990844e-06,
+      "loss": 0.04255459904670715,
+      "mean_token_accuracy": 0.9856055378913879,
+      "num_tokens": 2973148.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.04595456244423986,
+      "epoch": 2.647332617257429,
+      "grad_norm": 0.16675622761249542,
+      "learning_rate": 7.225869169754695e-06,
+      "loss": 0.04211015999317169,
+      "mean_token_accuracy": 0.9849406853318214,
+      "num_tokens": 2989259.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.04728242391720414,
+      "epoch": 2.661654135338346,
+      "grad_norm": 0.17016443610191345,
+      "learning_rate": 6.661315385496425e-06,
+      "loss": 0.04218058884143829,
+      "mean_token_accuracy": 0.9852652899920941,
+      "num_tokens": 3005497.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.04403044090140611,
+      "epoch": 2.6759756534192625,
+      "grad_norm": 0.11992475390434265,
+      "learning_rate": 6.118962510165038e-06,
+      "loss": 0.04122324287891388,
+      "mean_token_accuracy": 0.9865134075284004,
+      "num_tokens": 3021884.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.051198359485715626,
+      "epoch": 2.690297171500179,
+      "grad_norm": 0.12536704540252686,
+      "learning_rate": 5.598939544151327e-06,
+      "loss": 0.043464934825897215,
+      "mean_token_accuracy": 0.9847031898796559,
+      "num_tokens": 3037622.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.04911694664042443,
+      "epoch": 2.7046186895810957,
+      "grad_norm": 0.14136236906051636,
+      "learning_rate": 5.101370176604725e-06,
+      "loss": 0.043411648273468016,
+      "mean_token_accuracy": 0.9853925466537475,
+      "num_tokens": 3053821.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.047909668274223806,
+      "epoch": 2.718940207662012,
+      "grad_norm": 0.14949224889278412,
+      "learning_rate": 4.626372756013409e-06,
+      "loss": 0.04192754328250885,
+      "mean_token_accuracy": 0.9857831299304962,
+      "num_tokens": 3070199.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.04517021020874381,
+      "epoch": 2.7332617257429286,
+      "grad_norm": 0.1280951350927353,
+      "learning_rate": 4.174060262054769e-06,
+      "loss": 0.04178732335567474,
+      "mean_token_accuracy": 0.98523658066988,
+      "num_tokens": 3086555.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.04387184465304017,
+      "epoch": 2.7475832438238452,
+      "grad_norm": 0.1431456208229065,
+      "learning_rate": 3.7445402787227966e-06,
+      "loss": 0.042521944642066954,
+      "mean_token_accuracy": 0.9854860790073872,
+      "num_tokens": 3102751.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.046822808450087905,
+      "epoch": 2.761904761904762,
+      "grad_norm": 0.14817272126674652,
+      "learning_rate": 3.3379149687388867e-06,
+      "loss": 0.041089946031570436,
+      "mean_token_accuracy": 0.9857824392616749,
+      "num_tokens": 3119131.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.046427267813123764,
+      "epoch": 2.7762262799856785,
+      "grad_norm": 0.13534760475158691,
+      "learning_rate": 2.9542810492521054e-06,
+      "loss": 0.04210456609725952,
+      "mean_token_accuracy": 0.9851124472916126,
+      "num_tokens": 3135288.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.04964283937588334,
+      "epoch": 2.790547798066595,
+      "grad_norm": 0.1270522028207779,
+      "learning_rate": 2.5937297688347516e-06,
+      "loss": 0.0430122047662735,
+      "mean_token_accuracy": 0.9848847217857838,
+      "num_tokens": 3151534.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.04522125096991658,
+      "epoch": 2.804869316147512,
+      "grad_norm": 0.15303286910057068,
+      "learning_rate": 2.2563468857784818e-06,
+      "loss": 0.04267477989196777,
+      "mean_token_accuracy": 0.9850653804838657,
+      "num_tokens": 3167626.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.046864311979152265,
+      "epoch": 2.8191908342284284,
+      "grad_norm": 0.16102418303489685,
+      "learning_rate": 1.9422126476964843e-06,
+      "loss": 0.04267196953296661,
+      "mean_token_accuracy": 0.9853817634284496,
+      "num_tokens": 3183932.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.05009770723991096,
+      "epoch": 2.833512352309345,
+      "grad_norm": 0.15134263038635254,
+      "learning_rate": 1.6514017724363673e-06,
+      "loss": 0.04406825602054596,
+      "mean_token_accuracy": 0.9839926853775978,
+      "num_tokens": 3199902.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.04769130984786898,
+      "epoch": 2.8478338703902613,
+      "grad_norm": 0.1527557671070099,
+      "learning_rate": 1.383983430308189e-06,
+      "loss": 0.042158198356628415,
+      "mean_token_accuracy": 0.9859335079789162,
+      "num_tokens": 3216208.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.04732387030962855,
+      "epoch": 2.862155388471178,
+      "grad_norm": 0.13823670148849487,
+      "learning_rate": 1.1400212276321376e-06,
+      "loss": 0.041994097828865054,
+      "mean_token_accuracy": 0.9850150063633919,
+      "num_tokens": 3232527.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.04719096219632775,
+      "epoch": 2.8764769065520945,
+      "grad_norm": 0.1684579998254776,
+      "learning_rate": 9.195731916095241e-07,
+      "loss": 0.042640531063079835,
+      "mean_token_accuracy": 0.9859588980674744,
+      "num_tokens": 3248533.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.04635364734567702,
+      "epoch": 2.890798424633011,
+      "grad_norm": 0.13871508836746216,
+      "learning_rate": 7.226917565208657e-07,
+      "loss": 0.041817018389701845,
+      "mean_token_accuracy": 0.9853268563747406,
+      "num_tokens": 3264657.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.04832037026062608,
+      "epoch": 2.905119942713928,
+      "grad_norm": 0.13655678927898407,
+      "learning_rate": 5.494237512541855e-07,
+      "loss": 0.042303469777107236,
+      "mean_token_accuracy": 0.9850983947515488,
+      "num_tokens": 3280809.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.04568132129497826,
+      "epoch": 2.919441460794844,
+      "grad_norm": 0.12819744646549225,
+      "learning_rate": 3.998103881665993e-07,
+      "loss": 0.042490449547767636,
+      "mean_token_accuracy": 0.9860521554946899,
+      "num_tokens": 3297097.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.047166350809857246,
+      "epoch": 2.9337629788757607,
+      "grad_norm": 0.1366415172815323,
+      "learning_rate": 2.7388725328189036e-07,
+      "loss": 0.043278318643569944,
+      "mean_token_accuracy": 0.9850944019854069,
+      "num_tokens": 3313107.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.04601493317168206,
+      "epoch": 2.9480844969566773,
+      "grad_norm": 0.13710035383701324,
+      "learning_rate": 1.716842978261579e-07,
+      "loss": 0.042042380571365355,
+      "mean_token_accuracy": 0.9855021581053733,
+      "num_tokens": 3329427.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.04815540758427232,
+      "epoch": 2.962406015037594,
+      "grad_norm": 0.14303818345069885,
+      "learning_rate": 9.32258311039269e-08,
+      "loss": 0.0424267053604126,
+      "mean_token_accuracy": 0.9845395714044571,
+      "num_tokens": 3345553.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.045734380721114576,
+      "epoch": 2.9767275331185106,
+      "grad_norm": 0.1507130265235901,
+      "learning_rate": 3.8530514716017544e-08,
+      "loss": 0.04212013483047485,
+      "mean_token_accuracy": 0.9859001569449901,
+      "num_tokens": 3361923.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.045382294873707,
+      "epoch": 2.991049051199427,
+      "grad_norm": 0.11295795440673828,
+      "learning_rate": 7.611358120862555e-09,
+      "loss": 0.04327742159366608,
+      "mean_token_accuracy": 0.9849654823541641,
+      "num_tokens": 3377946.0,
+      "step": 2090
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.044319279225809236,
+      "eval_loss": 0.04254546016454697,
+      "eval_mean_token_accuracy": 0.9859963181067486,
+      "eval_num_tokens": 3388140.0,
+      "eval_runtime": 132.2104,
+      "eval_samples_per_second": 2.224,
+      "eval_steps_per_second": 2.224,
+      "step": 2097
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2097,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.001208219508736e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

lora_adapter/checkpoint-2097/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e3a9b47617458e67ed5dfbd51df40601253710b43c78072693271669fd135a
+size 5713

lora_adapter/checkpoint-699/README.md

@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Llama-3.2-1B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.21.0

lora_adapter/checkpoint-699/adapter_config.json

@@ -0,0 +1,51 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "kasa_config": null,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.21.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

lora_adapter/checkpoint-699/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ef3335540af112cb7215fb0019213f5ce61241adefa1978371bbd0c552dca29
+size 45118424

lora_adapter/checkpoint-699/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

lora_adapter/checkpoint-699/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00826bb5fbc679e4dceea56819e01ce3b603fc082137d8b5d2039ad4b12cc8d8
+size 90359883

lora_adapter/checkpoint-699/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3a77d4a8b98ce027a4d6a3b9fb5d7c904e27ec1efd5c0468c24fa26bb738316
+size 14455

lora_adapter/checkpoint-699/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca670fb7473c93e772629068cd8f70ffeb00fb0d9d777dc87d1ed9ce55f36603
+size 1465

lora_adapter/checkpoint-699/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

lora_adapter/checkpoint-699/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|finetune_right_pad_id|>",
+  "padding_side": "left",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null
+}

lora_adapter/checkpoint-699/trainer_state.json

@@ -0,0 +1,735 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 699,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 3.418184721469879,
+      "epoch": 0.014321518080916577,
+      "grad_norm": 5.165404319763184,
+      "learning_rate": 3e-05,
+      "loss": 4.75037727355957,
+      "mean_token_accuracy": 0.3666964743286371,
+      "num_tokens": 16368.0,
+      "step": 10
+    },
+    {
+      "entropy": 3.8264630317687987,
+      "epoch": 0.028643036161833155,
+      "grad_norm": 2.789987802505493,
+      "learning_rate": 6.333333333333333e-05,
+      "loss": 3.6756973266601562,
+      "mean_token_accuracy": 0.4501046933233738,
+      "num_tokens": 32587.0,
+      "step": 20
+    },
+    {
+      "entropy": 3.0331100687384605,
+      "epoch": 0.04296455424274973,
+      "grad_norm": 2.0175697803497314,
+      "learning_rate": 9.666666666666667e-05,
+      "loss": 2.153616523742676,
+      "mean_token_accuracy": 0.6448496542870998,
+      "num_tokens": 48627.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.7019604802131654,
+      "epoch": 0.05728607232366631,
+      "grad_norm": 1.4288526773452759,
+      "learning_rate": 0.00013000000000000002,
+      "loss": 1.1227694511413575,
+      "mean_token_accuracy": 0.8201614201068879,
+      "num_tokens": 64784.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.8430624332278966,
+      "epoch": 0.07160759040458288,
+      "grad_norm": 0.9607819318771362,
+      "learning_rate": 0.00016333333333333334,
+      "loss": 0.5127146720886231,
+      "mean_token_accuracy": 0.9230180129408836,
+      "num_tokens": 80994.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.3905979799106717,
+      "epoch": 0.08592910848549946,
+      "grad_norm": 0.8198261260986328,
+      "learning_rate": 0.00019666666666666666,
+      "loss": 0.3345895767211914,
+      "mean_token_accuracy": 0.9489256188273429,
+      "num_tokens": 97095.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.3071663906797767,
+      "epoch": 0.10025062656641603,
+      "grad_norm": 2.082451343536377,
+      "learning_rate": 0.00019999036690733865,
+      "loss": 0.29979329109191893,
+      "mean_token_accuracy": 0.9523964017629624,
+      "num_tokens": 113170.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.24958498124033213,
+      "epoch": 0.11457214464733262,
+      "grad_norm": 1.3423985242843628,
+      "learning_rate": 0.00019995706971049468,
+      "loss": 0.19564504623413087,
+      "mean_token_accuracy": 0.9659054979681969,
+      "num_tokens": 129473.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.16129616871476174,
+      "epoch": 0.1288936627282492,
+      "grad_norm": 0.6217024922370911,
+      "learning_rate": 0.00019989999740035867,
+      "loss": 0.1112561583518982,
+      "mean_token_accuracy": 0.9752564527094364,
+      "num_tokens": 145693.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.1593495703302324,
+      "epoch": 0.14321518080916576,
+      "grad_norm": 0.9386972784996033,
+      "learning_rate": 0.00019981916355176472,
+      "loss": 0.124180269241333,
+      "mean_token_accuracy": 0.9724250495433807,
+      "num_tokens": 161885.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.11440805536694824,
+      "epoch": 0.15753669889008234,
+      "grad_norm": 0.49029725790023804,
+      "learning_rate": 0.00019971458739130598,
+      "loss": 0.08359839916229247,
+      "mean_token_accuracy": 0.9790669329464435,
+      "num_tokens": 178194.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.16797136263921858,
+      "epoch": 0.17185821697099893,
+      "grad_norm": 0.4058690071105957,
+      "learning_rate": 0.00019958629379276138,
+      "loss": 0.14625961780548097,
+      "mean_token_accuracy": 0.9697998225688934,
+      "num_tokens": 194281.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.2288114172872156,
+      "epoch": 0.1861797350519155,
+      "grad_norm": 0.43757033348083496,
+      "learning_rate": 0.00019943431327117957,
+      "loss": 0.1752084255218506,
+      "mean_token_accuracy": 0.9670783936977386,
+      "num_tokens": 210353.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.14037263644859194,
+      "epoch": 0.20050125313283207,
+      "grad_norm": 0.4667147696018219,
+      "learning_rate": 0.00019925868197562062,
+      "loss": 0.09937882423400879,
+      "mean_token_accuracy": 0.9777047835290432,
+      "num_tokens": 226307.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.13454449912533165,
+      "epoch": 0.21482277121374865,
+      "grad_norm": 0.5024069547653198,
+      "learning_rate": 0.00019905944168055798,
+      "loss": 0.10526319742202758,
+      "mean_token_accuracy": 0.9757608838379384,
+      "num_tokens": 242366.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.10749808787368238,
+      "epoch": 0.22914428929466524,
+      "grad_norm": 0.3371286392211914,
+      "learning_rate": 0.00019883663977594208,
+      "loss": 0.07259818315505981,
+      "mean_token_accuracy": 0.9807105578482151,
+      "num_tokens": 258533.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.14817415219731628,
+      "epoch": 0.24346580737558182,
+      "grad_norm": 0.6096305251121521,
+      "learning_rate": 0.00019859032925592874,
+      "loss": 0.1135907769203186,
+      "mean_token_accuracy": 0.9745507664978504,
+      "num_tokens": 274810.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.1133763374760747,
+      "epoch": 0.2577873254564984,
+      "grad_norm": 0.41122370958328247,
+      "learning_rate": 0.00019832056870627417,
+      "loss": 0.09238196611404419,
+      "mean_token_accuracy": 0.977765066921711,
+      "num_tokens": 290922.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.07603299776092172,
+      "epoch": 0.272108843537415,
+      "grad_norm": 0.2798531949520111,
+      "learning_rate": 0.00019802742229040025,
+      "loss": 0.06427340507507324,
+      "mean_token_accuracy": 0.9813245065510273,
+      "num_tokens": 307118.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.11896687960252166,
+      "epoch": 0.2864303616183315,
+      "grad_norm": 0.24208419024944305,
+      "learning_rate": 0.00019771095973413293,
+      "loss": 0.10308414697647095,
+      "mean_token_accuracy": 0.9760065227746964,
+      "num_tokens": 323416.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.09079958228394389,
+      "epoch": 0.3007518796992481,
+      "grad_norm": 0.4041318893432617,
+      "learning_rate": 0.00019737125630911784,
+      "loss": 0.06735156178474426,
+      "mean_token_accuracy": 0.9801696293056011,
+      "num_tokens": 339621.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.09143369558732957,
+      "epoch": 0.3150733977801647,
+      "grad_norm": 0.3946863114833832,
+      "learning_rate": 0.00019700839281491662,
+      "loss": 0.07498582005500794,
+      "mean_token_accuracy": 0.9808824859559536,
+      "num_tokens": 355782.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.12254208489321172,
+      "epoch": 0.32939491586108127,
+      "grad_norm": 0.31145262718200684,
+      "learning_rate": 0.00019662245555978846,
+      "loss": 0.09450015425682068,
+      "mean_token_accuracy": 0.976518415659666,
+      "num_tokens": 371844.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.10461308304220437,
+      "epoch": 0.34371643394199786,
+      "grad_norm": 0.3476834297180176,
+      "learning_rate": 0.00019621353634016153,
+      "loss": 0.08276034593582153,
+      "mean_token_accuracy": 0.9795444570481777,
+      "num_tokens": 388015.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.11855657077394426,
+      "epoch": 0.35803795202291444,
+      "grad_norm": 0.32865747809410095,
+      "learning_rate": 0.00019578173241879872,
+      "loss": 0.09439409971237182,
+      "mean_token_accuracy": 0.9776719771325588,
+      "num_tokens": 404196.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.11360202138312162,
+      "epoch": 0.372359470103831,
+      "grad_norm": 0.41493919491767883,
+      "learning_rate": 0.0001953271465016635,
+      "loss": 0.09676467776298522,
+      "mean_token_accuracy": 0.9767774596810341,
+      "num_tokens": 420199.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.1050779873970896,
+      "epoch": 0.3866809881847476,
+      "grad_norm": 0.22015489637851715,
+      "learning_rate": 0.000194849886713491,
+      "loss": 0.07251381278038024,
+      "mean_token_accuracy": 0.9796808525919914,
+      "num_tokens": 436468.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.10228129292372615,
+      "epoch": 0.40100250626566414,
+      "grad_norm": 0.27469080686569214,
+      "learning_rate": 0.00019435006657206997,
+      "loss": 0.09110605716705322,
+      "mean_token_accuracy": 0.9781862519681453,
+      "num_tokens": 452720.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.09573564094025641,
+      "epoch": 0.4153240243465807,
+      "grad_norm": 0.20515574514865875,
+      "learning_rate": 0.00019382780496124253,
+      "loss": 0.06511516571044922,
+      "mean_token_accuracy": 0.9820534713566303,
+      "num_tokens": 468938.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.06887364573776722,
+      "epoch": 0.4296455424274973,
+      "grad_norm": 0.2742563784122467,
+      "learning_rate": 0.00019328322610262703,
+      "loss": 0.053380900621414186,
+      "mean_token_accuracy": 0.9827778674662113,
+      "num_tokens": 485200.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.10576252338942141,
+      "epoch": 0.4439670605084139,
+      "grad_norm": 0.22944959998130798,
+      "learning_rate": 0.00019271645952607163,
+      "loss": 0.09060812592506409,
+      "mean_token_accuracy": 0.9780363194644451,
+      "num_tokens": 501474.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.1117024946026504,
+      "epoch": 0.4582885785893305,
+      "grad_norm": 0.2669987678527832,
+      "learning_rate": 0.0001921276400388451,
+      "loss": 0.07678983211517335,
+      "mean_token_accuracy": 0.9798587985336781,
+      "num_tokens": 517580.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.10053576473146678,
+      "epoch": 0.47261009667024706,
+      "grad_norm": 0.24858009815216064,
+      "learning_rate": 0.00019151690769357265,
+      "loss": 0.08534708023071289,
+      "mean_token_accuracy": 0.9798071250319481,
+      "num_tokens": 533788.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.13230610005557536,
+      "epoch": 0.48693161475116364,
+      "grad_norm": 0.2949056923389435,
+      "learning_rate": 0.00019088440775492384,
+      "loss": 0.09592426419258118,
+      "mean_token_accuracy": 0.9757725939154624,
+      "num_tokens": 549966.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.06407897693570704,
+      "epoch": 0.5012531328320802,
+      "grad_norm": 0.24437105655670166,
+      "learning_rate": 0.00019023029066506087,
+      "loss": 0.05607050061225891,
+      "mean_token_accuracy": 0.9830520637333393,
+      "num_tokens": 566066.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.06379525621887297,
+      "epoch": 0.5155746509129968,
+      "grad_norm": 0.2406875491142273,
+      "learning_rate": 0.00018955471200785551,
+      "loss": 0.05550529956817627,
+      "mean_token_accuracy": 0.9826574549078941,
+      "num_tokens": 582036.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.10768530536442995,
+      "epoch": 0.5298961689939133,
+      "grad_norm": 0.19389888644218445,
+      "learning_rate": 0.00018885783247188305,
+      "loss": 0.09576615095138549,
+      "mean_token_accuracy": 0.9779540851712227,
+      "num_tokens": 598182.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.12297249888069928,
+      "epoch": 0.54421768707483,
+      "grad_norm": 0.33548423647880554,
+      "learning_rate": 0.0001881398178122017,
+      "loss": 0.08422422409057617,
+      "mean_token_accuracy": 0.9791318334639072,
+      "num_tokens": 614468.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.06059104287996888,
+      "epoch": 0.5585392051557465,
+      "grad_norm": 0.34327495098114014,
+      "learning_rate": 0.0001874008388109276,
+      "loss": 0.05207563042640686,
+      "mean_token_accuracy": 0.9843599937856198,
+      "num_tokens": 630800.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.09479554321151226,
+      "epoch": 0.572860723236663,
+      "grad_norm": 0.3410899043083191,
+      "learning_rate": 0.00018664107123661337,
+      "loss": 0.0662381649017334,
+      "mean_token_accuracy": 0.9813465528190136,
+      "num_tokens": 646956.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.09436679913196713,
+      "epoch": 0.5871822413175797,
+      "grad_norm": 0.8792180418968201,
+      "learning_rate": 0.00018586069580244128,
+      "loss": 0.07904249429702759,
+      "mean_token_accuracy": 0.9798643849790096,
+      "num_tokens": 663240.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.09691147103440016,
+      "epoch": 0.6015037593984962,
+      "grad_norm": 0.23540320992469788,
+      "learning_rate": 0.00018505989812323993,
+      "loss": 0.06907619833946228,
+      "mean_token_accuracy": 0.9815059810876846,
+      "num_tokens": 679484.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.07770795098040253,
+      "epoch": 0.6158252774794128,
+      "grad_norm": 0.28902187943458557,
+      "learning_rate": 0.0001842388686713351,
+      "loss": 0.05818814635276794,
+      "mean_token_accuracy": 0.9836779788136483,
+      "num_tokens": 695720.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.06032483666203916,
+      "epoch": 0.6301467955603294,
+      "grad_norm": 0.328427791595459,
+      "learning_rate": 0.0001833978027312453,
+      "loss": 0.053039717674255374,
+      "mean_token_accuracy": 0.9840192586183548,
+      "num_tokens": 711945.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.0849465676350519,
+      "epoch": 0.644468313641246,
+      "grad_norm": 0.16869916021823883,
+      "learning_rate": 0.00018253690035323285,
+      "loss": 0.06781792044639587,
+      "mean_token_accuracy": 0.9806929357349873,
+      "num_tokens": 728029.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.08969353097490966,
+      "epoch": 0.6587898317221625,
+      "grad_norm": 0.18590566515922546,
+      "learning_rate": 0.0001816563663057211,
+      "loss": 0.06440567970275879,
+      "mean_token_accuracy": 0.9825256459414959,
+      "num_tokens": 744330.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.06217158674262464,
+      "epoch": 0.6731113498030791,
+      "grad_norm": 0.27869918942451477,
+      "learning_rate": 0.00018075641002658976,
+      "loss": 0.053045254945755,
+      "mean_token_accuracy": 0.9843784637749196,
+      "num_tokens": 760323.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.0674417833564803,
+      "epoch": 0.6874328678839957,
+      "grad_norm": 0.1830916553735733,
+      "learning_rate": 0.00017983724557335925,
+      "loss": 0.055755794048309326,
+      "mean_token_accuracy": 0.9838082902133465,
+      "num_tokens": 776628.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.0802982683526352,
+      "epoch": 0.7017543859649122,
+      "grad_norm": 0.4552997350692749,
+      "learning_rate": 0.00017889909157227653,
+      "loss": 0.060858166217803954,
+      "mean_token_accuracy": 0.9824069820344448,
+      "num_tokens": 792838.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.07723174102138727,
+      "epoch": 0.7160759040458289,
+      "grad_norm": 0.18987013399600983,
+      "learning_rate": 0.00017794217116631417,
+      "loss": 0.060531008243560794,
+      "mean_token_accuracy": 0.9814355976879596,
+      "num_tokens": 808999.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.05520166652277112,
+      "epoch": 0.7303974221267454,
+      "grad_norm": 0.18790723383426666,
+      "learning_rate": 0.00017696671196209496,
+      "loss": 0.05018657445907593,
+      "mean_token_accuracy": 0.983309705555439,
+      "num_tokens": 825062.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.05418659960851073,
+      "epoch": 0.744718940207662,
+      "grad_norm": 0.160777747631073,
+      "learning_rate": 0.00017597294597575507,
+      "loss": 0.04926420748233795,
+      "mean_token_accuracy": 0.9841075465083122,
+      "num_tokens": 841241.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.1152086689369753,
+      "epoch": 0.7590404582885786,
+      "grad_norm": 0.6535390615463257,
+      "learning_rate": 0.0001749611095777581,
+      "loss": 0.0968184232711792,
+      "mean_token_accuracy": 0.975805052369833,
+      "num_tokens": 857300.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.07658788892440498,
+      "epoch": 0.7733619763694952,
+      "grad_norm": 0.1728331297636032,
+      "learning_rate": 0.00017393144343667364,
+      "loss": 0.06037890911102295,
+      "mean_token_accuracy": 0.9823231592774391,
+      "num_tokens": 873434.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.09907218038570136,
+      "epoch": 0.7876834944504117,
+      "grad_norm": 0.22389887273311615,
+      "learning_rate": 0.00017288419246193373,
+      "loss": 0.07112311124801636,
+      "mean_token_accuracy": 0.9803691074252129,
+      "num_tokens": 889567.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.05675842456985265,
+      "epoch": 0.8020050125313283,
+      "grad_norm": 0.17525967955589294,
+      "learning_rate": 0.00017181960574557992,
+      "loss": 0.04961739480495453,
+      "mean_token_accuracy": 0.9838187299668789,
+      "num_tokens": 905625.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.08107102129142732,
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.17234288156032562,
+      "learning_rate": 0.00017073793650301624,
+      "loss": 0.06352313160896302,
+      "mean_token_accuracy": 0.9820462837815285,
+      "num_tokens": 921757.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.061689483653754,
+      "epoch": 0.8306480486931614,
+      "grad_norm": 0.23366186022758484,
+      "learning_rate": 0.00016963944201278097,
+      "loss": 0.051079404354095456,
+      "mean_token_accuracy": 0.982898436486721,
+      "num_tokens": 937984.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.06318069302942604,
+      "epoch": 0.8449695667740781,
+      "grad_norm": 0.19096462428569794,
+      "learning_rate": 0.00016852438355535214,
+      "loss": 0.05602290034294129,
+      "mean_token_accuracy": 0.9843950115144253,
+      "num_tokens": 954323.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.05533521031029522,
+      "epoch": 0.8592910848549946,
+      "grad_norm": 0.21601422131061554,
+      "learning_rate": 0.00016739302635100108,
+      "loss": 0.04836077392101288,
+      "mean_token_accuracy": 0.9856260627508163,
+      "num_tokens": 970505.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.05332150736358017,
+      "epoch": 0.8736126029359113,
+      "grad_norm": 0.16544337570667267,
+      "learning_rate": 0.00016624563949670904,
+      "loss": 0.04835851788520813,
+      "mean_token_accuracy": 0.9843276873230934,
+      "num_tokens": 986682.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.052138336235657334,
+      "epoch": 0.8879341210168278,
+      "grad_norm": 0.1802208125591278,
+      "learning_rate": 0.0001650824959021614,
+      "loss": 0.05234106183052063,
+      "mean_token_accuracy": 0.9847004011273384,
+      "num_tokens": 1002790.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.055844594305381176,
+      "epoch": 0.9022556390977443,
+      "grad_norm": 0.15213149785995483,
+      "learning_rate": 0.00016390387222483537,
+      "loss": 0.04984959363937378,
+      "mean_token_accuracy": 0.983958077430725,
+      "num_tokens": 1019089.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.06128789323847741,
+      "epoch": 0.916577157178661,
+      "grad_norm": 0.12318123877048492,
+      "learning_rate": 0.00016271004880419608,
+      "loss": 0.05124086737632751,
+      "mean_token_accuracy": 0.9843113601207734,
+      "num_tokens": 1035332.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.0652699463069439,
+      "epoch": 0.9308986752595775,
+      "grad_norm": 0.16328729689121246,
+      "learning_rate": 0.00016150130959501687,
+      "loss": 0.051763099431991574,
+      "mean_token_accuracy": 0.9837749816477299,
+      "num_tokens": 1051331.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.0749959686305374,
+      "epoch": 0.9452201933404941,
+      "grad_norm": 0.2559461295604706,
+      "learning_rate": 0.0001602779420998398,
+      "loss": 0.05571107268333435,
+      "mean_token_accuracy": 0.9834148533642292,
+      "num_tokens": 1067608.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.05268463878892362,
+      "epoch": 0.9595417114214106,
+      "grad_norm": 0.14703021943569183,
+      "learning_rate": 0.00015904023730059228,
+      "loss": 0.04806509017944336,
+      "mean_token_accuracy": 0.9848555572330951,
+      "num_tokens": 1083704.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.08527488259132951,
+      "epoch": 0.9738632295023273,
+      "grad_norm": 0.27216821908950806,
+      "learning_rate": 0.00015778848958937586,
+      "loss": 0.061145973205566403,
+      "mean_token_accuracy": 0.9831270687282085,
+      "num_tokens": 1099812.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.05362487006932497,
+      "epoch": 0.9881847475832438,
+      "grad_norm": 0.20429278910160065,
+      "learning_rate": 0.00015652299669844438,
+      "loss": 0.04693828225135803,
+      "mean_token_accuracy": 0.9854582466185093,
+      "num_tokens": 1116039.0,
+      "step": 690
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 0.05509232616267439,
+      "eval_loss": 0.04924684017896652,
+      "eval_mean_token_accuracy": 0.9852213520987504,
+      "eval_num_tokens": 1129380.0,
+      "eval_runtime": 132.3751,
+      "eval_samples_per_second": 2.221,
+      "eval_steps_per_second": 2.221,
+      "step": 699
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2097,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6670694065029120.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

lora_adapter/checkpoint-699/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e3a9b47617458e67ed5dfbd51df40601253710b43c78072693271669fd135a
+size 5713

lora_adapter/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

lora_adapter/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|finetune_right_pad_id|>",
+  "padding_side": "left",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null
+}

lora_adapter/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e3a9b47617458e67ed5dfbd51df40601253710b43c78072693271669fd135a
+size 5713