diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a4ccc99d12a1f1db0badf20df3dae4edc3cb43d2 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: transformers +model_name: Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3-4B-Base](https://huggingface.co/Qwen/Qwen3-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/7kp8vcdh) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fe15fc4684cbc8b0f9906f045294ddc5e8d46063 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.07029303730572957, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5d4bb4dcf645336623e7b86d22735f4f996f2f15 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1266/trainer_state.json @@ -0,0 +1,317 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1266, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.577122880220413, + "epoch": 0.11862396204033215, + "grad_norm": 1.4608460664749146, + "learning_rate": 3.051949723346327e-05, + "loss": 2.5035160827636718, + "mean_token_accuracy": 0.556775416135788, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 1.0250165474414825, + "epoch": 0.2372479240806643, + "grad_norm": 0.9524351954460144, + "learning_rate": 6.166184134924211e-05, + "loss": 0.9732361602783203, + "mean_token_accuracy": 0.747636867761612, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 0.8675219911336899, + "epoch": 0.35587188612099646, + "grad_norm": 0.7895523905754089, + "learning_rate": 9.280418546502096e-05, + "loss": 0.8090463256835938, + "mean_token_accuracy": 0.7829809945821762, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.8144319581985474, + "epoch": 0.4744958481613286, + "grad_norm": 0.6939403414726257, + "learning_rate": 0.0001239465295807998, + "loss": 0.7548302459716797, + "mean_token_accuracy": 0.7972425794601441, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.791321137547493, + "epoch": 0.5931198102016607, + "grad_norm": 0.6105369925498962, + "learning_rate": 0.00015508887369657866, + "loss": 0.7202831268310547, + "mean_token_accuracy": 0.8030376029014588, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.7602744549512863, + "epoch": 0.7117437722419929, + "grad_norm": 0.696577250957489, + "learning_rate": 0.0001862312178123575, + "loss": 0.6942585754394531, + "mean_token_accuracy": 0.8084166407585144, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.732809653878212, + "epoch": 0.830367734282325, + "grad_norm": 0.5414506196975708, + "learning_rate": 0.00021737356192813635, + "loss": 0.672747573852539, + "mean_token_accuracy": 0.8152892899513244, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7336753070354461, + "epoch": 0.9489916963226572, + "grad_norm": 0.46377503871917725, + "learning_rate": 0.0002485159060439152, + "loss": 0.6722122192382812, + "mean_token_accuracy": 0.8157827919721603, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6210183713868348, + "eval_loss": 0.6189655661582947, + "eval_mean_token_accuracy": 0.8275951066909478, + "eval_num_tokens": 668662.0, + "eval_runtime": 70.4547, + "eval_samples_per_second": 19.36, + "eval_steps_per_second": 2.427, + "step": 422 + }, + { + "entropy": 0.7184271240475202, + "epoch": 1.066429418742586, + "grad_norm": 0.44397908449172974, + "learning_rate": 0.00026280861006503046, + "loss": 0.6535201263427735, + "mean_token_accuracy": 0.814659362489527, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.6942325741052627, + "epoch": 1.1850533807829182, + "grad_norm": 0.4297226667404175, + "learning_rate": 0.00026257490827119015, + "loss": 0.6245469665527343, + "mean_token_accuracy": 0.8250543695688247, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.6845822006464004, + "epoch": 1.3036773428232502, + "grad_norm": 0.5480059385299683, + "learning_rate": 0.00026211689567831744, + "loss": 0.6164654922485352, + "mean_token_accuracy": 0.8266578191518783, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.697658548951149, + "epoch": 1.4223013048635824, + "grad_norm": 0.6630405187606812, + "learning_rate": 0.00026143535561746887, + "loss": 0.6209470367431641, + "mean_token_accuracy": 0.8271088349819183, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6747388768196106, + "epoch": 1.5409252669039146, + "grad_norm": 0.6049144268035889, + "learning_rate": 0.00026053145371488325, + "loss": 0.6101105880737304, + "mean_token_accuracy": 0.8299184310436248, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6651581016182899, + "epoch": 1.6595492289442468, + "grad_norm": 0.4120999276638031, + "learning_rate": 0.0002594067358984311, + "loss": 0.5947869873046875, + "mean_token_accuracy": 0.8315655797719955, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6637684524059295, + "epoch": 1.778173190984579, + "grad_norm": 0.4927777349948883, + "learning_rate": 0.00025806312575364106, + "loss": 0.5965091323852539, + "mean_token_accuracy": 0.8311642789840699, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6609975713491439, + "epoch": 1.896797153024911, + "grad_norm": 0.3883759379386902, + "learning_rate": 0.0002565029212338241, + "loss": 0.5957939147949218, + "mean_token_accuracy": 0.8327557492256165, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5433656487897126, + "eval_loss": 0.5806400775909424, + "eval_mean_token_accuracy": 0.8379280535798324, + "eval_num_tokens": 1337324.0, + "eval_runtime": 70.0138, + "eval_samples_per_second": 19.482, + "eval_steps_per_second": 2.442, + "step": 844 + }, + { + "entropy": 0.6604163893545517, + "epoch": 2.01423487544484, + "grad_norm": 0.48883888125419617, + "learning_rate": 0.0002547287907299242, + "loss": 0.5921520614624023, + "mean_token_accuracy": 0.8313083654702312, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.5899698203802108, + "epoch": 2.132858837485172, + "grad_norm": 0.4636650085449219, + "learning_rate": 0.0002527437685068147, + "loss": 0.5210755538940429, + "mean_token_accuracy": 0.8464557534456253, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6036705967783927, + "epoch": 2.2514827995255042, + "grad_norm": 0.4339908957481384, + "learning_rate": 0.0002505512495138475, + "loss": 0.5236929702758789, + "mean_token_accuracy": 0.8439281392097473, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.5957452219724655, + "epoch": 2.3701067615658364, + "grad_norm": 0.4933948218822479, + "learning_rate": 0.0002481549835785298, + "loss": 0.5212778854370117, + "mean_token_accuracy": 0.8468212288618088, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.5991016891598702, + "epoch": 2.488730723606168, + "grad_norm": 0.4078773558139801, + "learning_rate": 0.0002455590689932581, + "loss": 0.5267618560791015, + "mean_token_accuracy": 0.8449788075685501, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.5956563407182693, + "epoch": 2.6073546856465004, + "grad_norm": 0.36706778407096863, + "learning_rate": 0.00024276794550607983, + "loss": 0.5267750549316407, + "mean_token_accuracy": 0.8461971461772919, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.5945340126752854, + "epoch": 2.7259786476868326, + "grad_norm": 0.37087908387184143, + "learning_rate": 0.0002397863867274683, + "loss": 0.522254867553711, + "mean_token_accuracy": 0.8455314397811889, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.5698307737708092, + "epoch": 2.844602609727165, + "grad_norm": 0.4367637038230896, + "learning_rate": 0.00023661949196609894, + "loss": 0.5050308609008789, + "mean_token_accuracy": 0.8516230022907257, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.5873846638202668, + "epoch": 2.963226571767497, + "grad_norm": 0.34372401237487793, + "learning_rate": 0.00023327267750758942, + "loss": 0.5120795440673828, + "mean_token_accuracy": 0.8469690150022506, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4637417015973587, + "eval_loss": 0.5590130090713501, + "eval_mean_token_accuracy": 0.8415542060171651, + "eval_num_tokens": 2005986.0, + "eval_runtime": 69.9647, + "eval_samples_per_second": 19.496, + "eval_steps_per_second": 2.444, + "step": 1266 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.291660257930035e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fe15fc4684cbc8b0f9906f045294ddc5e8d46063 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.07029303730572957, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a0607b51448751095d0f9579a556920cc3ca5d2f --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-1688/trainer_state.json @@ -0,0 +1,408 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1688, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.577122880220413, + "epoch": 0.11862396204033215, + "grad_norm": 1.4608460664749146, + "learning_rate": 3.051949723346327e-05, + "loss": 2.5035160827636718, + "mean_token_accuracy": 0.556775416135788, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 1.0250165474414825, + "epoch": 0.2372479240806643, + "grad_norm": 0.9524351954460144, + "learning_rate": 6.166184134924211e-05, + "loss": 0.9732361602783203, + "mean_token_accuracy": 0.747636867761612, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 0.8675219911336899, + "epoch": 0.35587188612099646, + "grad_norm": 0.7895523905754089, + "learning_rate": 9.280418546502096e-05, + "loss": 0.8090463256835938, + "mean_token_accuracy": 0.7829809945821762, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.8144319581985474, + "epoch": 0.4744958481613286, + "grad_norm": 0.6939403414726257, + "learning_rate": 0.0001239465295807998, + "loss": 0.7548302459716797, + "mean_token_accuracy": 0.7972425794601441, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.791321137547493, + "epoch": 0.5931198102016607, + "grad_norm": 0.6105369925498962, + "learning_rate": 0.00015508887369657866, + "loss": 0.7202831268310547, + "mean_token_accuracy": 0.8030376029014588, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.7602744549512863, + "epoch": 0.7117437722419929, + "grad_norm": 0.696577250957489, + "learning_rate": 0.0001862312178123575, + "loss": 0.6942585754394531, + "mean_token_accuracy": 0.8084166407585144, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.732809653878212, + "epoch": 0.830367734282325, + "grad_norm": 0.5414506196975708, + "learning_rate": 0.00021737356192813635, + "loss": 0.672747573852539, + "mean_token_accuracy": 0.8152892899513244, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7336753070354461, + "epoch": 0.9489916963226572, + "grad_norm": 0.46377503871917725, + "learning_rate": 0.0002485159060439152, + "loss": 0.6722122192382812, + "mean_token_accuracy": 0.8157827919721603, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6210183713868348, + "eval_loss": 0.6189655661582947, + "eval_mean_token_accuracy": 0.8275951066909478, + "eval_num_tokens": 668662.0, + "eval_runtime": 70.4547, + "eval_samples_per_second": 19.36, + "eval_steps_per_second": 2.427, + "step": 422 + }, + { + "entropy": 0.7184271240475202, + "epoch": 1.066429418742586, + "grad_norm": 0.44397908449172974, + "learning_rate": 0.00026280861006503046, + "loss": 0.6535201263427735, + "mean_token_accuracy": 0.814659362489527, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.6942325741052627, + "epoch": 1.1850533807829182, + "grad_norm": 0.4297226667404175, + "learning_rate": 0.00026257490827119015, + "loss": 0.6245469665527343, + "mean_token_accuracy": 0.8250543695688247, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.6845822006464004, + "epoch": 1.3036773428232502, + "grad_norm": 0.5480059385299683, + "learning_rate": 0.00026211689567831744, + "loss": 0.6164654922485352, + "mean_token_accuracy": 0.8266578191518783, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.697658548951149, + "epoch": 1.4223013048635824, + "grad_norm": 0.6630405187606812, + "learning_rate": 0.00026143535561746887, + "loss": 0.6209470367431641, + "mean_token_accuracy": 0.8271088349819183, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6747388768196106, + "epoch": 1.5409252669039146, + "grad_norm": 0.6049144268035889, + "learning_rate": 0.00026053145371488325, + "loss": 0.6101105880737304, + "mean_token_accuracy": 0.8299184310436248, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6651581016182899, + "epoch": 1.6595492289442468, + "grad_norm": 0.4120999276638031, + "learning_rate": 0.0002594067358984311, + "loss": 0.5947869873046875, + "mean_token_accuracy": 0.8315655797719955, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6637684524059295, + "epoch": 1.778173190984579, + "grad_norm": 0.4927777349948883, + "learning_rate": 0.00025806312575364106, + "loss": 0.5965091323852539, + "mean_token_accuracy": 0.8311642789840699, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6609975713491439, + "epoch": 1.896797153024911, + "grad_norm": 0.3883759379386902, + "learning_rate": 0.0002565029212338241, + "loss": 0.5957939147949218, + "mean_token_accuracy": 0.8327557492256165, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5433656487897126, + "eval_loss": 0.5806400775909424, + "eval_mean_token_accuracy": 0.8379280535798324, + "eval_num_tokens": 1337324.0, + "eval_runtime": 70.0138, + "eval_samples_per_second": 19.482, + "eval_steps_per_second": 2.442, + "step": 844 + }, + { + "entropy": 0.6604163893545517, + "epoch": 2.01423487544484, + "grad_norm": 0.48883888125419617, + "learning_rate": 0.0002547287907299242, + "loss": 0.5921520614624023, + "mean_token_accuracy": 0.8313083654702312, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.5899698203802108, + "epoch": 2.132858837485172, + "grad_norm": 0.4636650085449219, + "learning_rate": 0.0002527437685068147, + "loss": 0.5210755538940429, + "mean_token_accuracy": 0.8464557534456253, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6036705967783927, + "epoch": 2.2514827995255042, + "grad_norm": 0.4339908957481384, + "learning_rate": 0.0002505512495138475, + "loss": 0.5236929702758789, + "mean_token_accuracy": 0.8439281392097473, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.5957452219724655, + "epoch": 2.3701067615658364, + "grad_norm": 0.4933948218822479, + "learning_rate": 0.0002481549835785298, + "loss": 0.5212778854370117, + "mean_token_accuracy": 0.8468212288618088, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.5991016891598702, + "epoch": 2.488730723606168, + "grad_norm": 0.4078773558139801, + "learning_rate": 0.0002455590689932581, + "loss": 0.5267618560791015, + "mean_token_accuracy": 0.8449788075685501, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.5956563407182693, + "epoch": 2.6073546856465004, + "grad_norm": 0.36706778407096863, + "learning_rate": 0.00024276794550607983, + "loss": 0.5267750549316407, + "mean_token_accuracy": 0.8461971461772919, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.5945340126752854, + "epoch": 2.7259786476868326, + "grad_norm": 0.37087908387184143, + "learning_rate": 0.0002397863867274683, + "loss": 0.522254867553711, + "mean_token_accuracy": 0.8455314397811889, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.5698307737708092, + "epoch": 2.844602609727165, + "grad_norm": 0.4367637038230896, + "learning_rate": 0.00023661949196609894, + "loss": 0.5050308609008789, + "mean_token_accuracy": 0.8516230022907257, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.5873846638202668, + "epoch": 2.963226571767497, + "grad_norm": 0.34372401237487793, + "learning_rate": 0.00023327267750758942, + "loss": 0.5120795440673828, + "mean_token_accuracy": 0.8469690150022506, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4637417015973587, + "eval_loss": 0.5590130090713501, + "eval_mean_token_accuracy": 0.8415542060171651, + "eval_num_tokens": 2005986.0, + "eval_runtime": 69.9647, + "eval_samples_per_second": 19.496, + "eval_steps_per_second": 2.444, + "step": 1266 + }, + { + "entropy": 0.5193097814164981, + "epoch": 3.080664294187426, + "grad_norm": 0.4883718192577362, + "learning_rate": 0.00022975166735111956, + "loss": 0.4416360855102539, + "mean_token_accuracy": 0.8627295740927109, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.4912916052341461, + "epoch": 3.199288256227758, + "grad_norm": 0.42061012983322144, + "learning_rate": 0.00022606248341977406, + "loss": 0.41719051361083986, + "mean_token_accuracy": 0.8705551433563232, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.49565069168806075, + "epoch": 3.31791221826809, + "grad_norm": 0.4688358008861542, + "learning_rate": 0.00022221143526135096, + "loss": 0.425596809387207, + "mean_token_accuracy": 0.8679776644706726, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5045917698740959, + "epoch": 3.4365361803084222, + "grad_norm": 0.4515043795108795, + "learning_rate": 0.0002182051092572507, + "loss": 0.432201042175293, + "mean_token_accuracy": 0.867100123167038, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5014845252037048, + "epoch": 3.5551601423487544, + "grad_norm": 0.5356592535972595, + "learning_rate": 0.00021405035735790138, + "loss": 0.43230159759521486, + "mean_token_accuracy": 0.868084819316864, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.5087948343157769, + "epoch": 3.6737841043890866, + "grad_norm": 0.5144466757774353, + "learning_rate": 0.00020975428536398563, + "loss": 0.4351469039916992, + "mean_token_accuracy": 0.8663349187374115, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5100917038321495, + "epoch": 3.792408066429419, + "grad_norm": 0.39870721101760864, + "learning_rate": 0.00020532424077351232, + "loss": 0.4383781051635742, + "mean_token_accuracy": 0.8638313174247741, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.5078894594311714, + "epoch": 3.9110320284697506, + "grad_norm": 0.5162035226821899, + "learning_rate": 0.00020076780021551676, + "loss": 0.43954910278320314, + "mean_token_accuracy": 0.8654858589172363, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.42526503421409784, + "eval_loss": 0.5683684349060059, + "eval_mean_token_accuracy": 0.8431169801985311, + "eval_num_tokens": 2674648.0, + "eval_runtime": 69.9381, + "eval_samples_per_second": 19.503, + "eval_steps_per_second": 2.445, + "step": 1688 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2402310441946726e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fe15fc4684cbc8b0f9906f045294ddc5e8d46063 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.07029303730572957, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9889b89432d4f872835be82010e21ae3c44f1e29 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2110/trainer_state.json @@ -0,0 +1,509 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2110, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.577122880220413, + "epoch": 0.11862396204033215, + "grad_norm": 1.4608460664749146, + "learning_rate": 3.051949723346327e-05, + "loss": 2.5035160827636718, + "mean_token_accuracy": 0.556775416135788, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 1.0250165474414825, + "epoch": 0.2372479240806643, + "grad_norm": 0.9524351954460144, + "learning_rate": 6.166184134924211e-05, + "loss": 0.9732361602783203, + "mean_token_accuracy": 0.747636867761612, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 0.8675219911336899, + "epoch": 0.35587188612099646, + "grad_norm": 0.7895523905754089, + "learning_rate": 9.280418546502096e-05, + "loss": 0.8090463256835938, + "mean_token_accuracy": 0.7829809945821762, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.8144319581985474, + "epoch": 0.4744958481613286, + "grad_norm": 0.6939403414726257, + "learning_rate": 0.0001239465295807998, + "loss": 0.7548302459716797, + "mean_token_accuracy": 0.7972425794601441, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.791321137547493, + "epoch": 0.5931198102016607, + "grad_norm": 0.6105369925498962, + "learning_rate": 0.00015508887369657866, + "loss": 0.7202831268310547, + "mean_token_accuracy": 0.8030376029014588, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.7602744549512863, + "epoch": 0.7117437722419929, + "grad_norm": 0.696577250957489, + "learning_rate": 0.0001862312178123575, + "loss": 0.6942585754394531, + "mean_token_accuracy": 0.8084166407585144, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.732809653878212, + "epoch": 0.830367734282325, + "grad_norm": 0.5414506196975708, + "learning_rate": 0.00021737356192813635, + "loss": 0.672747573852539, + "mean_token_accuracy": 0.8152892899513244, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7336753070354461, + "epoch": 0.9489916963226572, + "grad_norm": 0.46377503871917725, + "learning_rate": 0.0002485159060439152, + "loss": 0.6722122192382812, + "mean_token_accuracy": 0.8157827919721603, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6210183713868348, + "eval_loss": 0.6189655661582947, + "eval_mean_token_accuracy": 0.8275951066909478, + "eval_num_tokens": 668662.0, + "eval_runtime": 70.4547, + "eval_samples_per_second": 19.36, + "eval_steps_per_second": 2.427, + "step": 422 + }, + { + "entropy": 0.7184271240475202, + "epoch": 1.066429418742586, + "grad_norm": 0.44397908449172974, + "learning_rate": 0.00026280861006503046, + "loss": 0.6535201263427735, + "mean_token_accuracy": 0.814659362489527, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.6942325741052627, + "epoch": 1.1850533807829182, + "grad_norm": 0.4297226667404175, + "learning_rate": 0.00026257490827119015, + "loss": 0.6245469665527343, + "mean_token_accuracy": 0.8250543695688247, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.6845822006464004, + "epoch": 1.3036773428232502, + "grad_norm": 0.5480059385299683, + "learning_rate": 0.00026211689567831744, + "loss": 0.6164654922485352, + "mean_token_accuracy": 0.8266578191518783, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.697658548951149, + "epoch": 1.4223013048635824, + "grad_norm": 0.6630405187606812, + "learning_rate": 0.00026143535561746887, + "loss": 0.6209470367431641, + "mean_token_accuracy": 0.8271088349819183, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6747388768196106, + "epoch": 1.5409252669039146, + "grad_norm": 0.6049144268035889, + "learning_rate": 0.00026053145371488325, + "loss": 0.6101105880737304, + "mean_token_accuracy": 0.8299184310436248, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6651581016182899, + "epoch": 1.6595492289442468, + "grad_norm": 0.4120999276638031, + "learning_rate": 0.0002594067358984311, + "loss": 0.5947869873046875, + "mean_token_accuracy": 0.8315655797719955, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6637684524059295, + "epoch": 1.778173190984579, + "grad_norm": 0.4927777349948883, + "learning_rate": 0.00025806312575364106, + "loss": 0.5965091323852539, + "mean_token_accuracy": 0.8311642789840699, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6609975713491439, + "epoch": 1.896797153024911, + "grad_norm": 0.3883759379386902, + "learning_rate": 0.0002565029212338241, + "loss": 0.5957939147949218, + "mean_token_accuracy": 0.8327557492256165, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5433656487897126, + "eval_loss": 0.5806400775909424, + "eval_mean_token_accuracy": 0.8379280535798324, + "eval_num_tokens": 1337324.0, + "eval_runtime": 70.0138, + "eval_samples_per_second": 19.482, + "eval_steps_per_second": 2.442, + "step": 844 + }, + { + "entropy": 0.6604163893545517, + "epoch": 2.01423487544484, + "grad_norm": 0.48883888125419617, + "learning_rate": 0.0002547287907299242, + "loss": 0.5921520614624023, + "mean_token_accuracy": 0.8313083654702312, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.5899698203802108, + "epoch": 2.132858837485172, + "grad_norm": 0.4636650085449219, + "learning_rate": 0.0002527437685068147, + "loss": 0.5210755538940429, + "mean_token_accuracy": 0.8464557534456253, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6036705967783927, + "epoch": 2.2514827995255042, + "grad_norm": 0.4339908957481384, + "learning_rate": 0.0002505512495138475, + "loss": 0.5236929702758789, + "mean_token_accuracy": 0.8439281392097473, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.5957452219724655, + "epoch": 2.3701067615658364, + "grad_norm": 0.4933948218822479, + "learning_rate": 0.0002481549835785298, + "loss": 0.5212778854370117, + "mean_token_accuracy": 0.8468212288618088, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.5991016891598702, + "epoch": 2.488730723606168, + "grad_norm": 0.4078773558139801, + "learning_rate": 0.0002455590689932581, + "loss": 0.5267618560791015, + "mean_token_accuracy": 0.8449788075685501, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.5956563407182693, + "epoch": 2.6073546856465004, + "grad_norm": 0.36706778407096863, + "learning_rate": 0.00024276794550607983, + "loss": 0.5267750549316407, + "mean_token_accuracy": 0.8461971461772919, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.5945340126752854, + "epoch": 2.7259786476868326, + "grad_norm": 0.37087908387184143, + "learning_rate": 0.0002397863867274683, + "loss": 0.522254867553711, + "mean_token_accuracy": 0.8455314397811889, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.5698307737708092, + "epoch": 2.844602609727165, + "grad_norm": 0.4367637038230896, + "learning_rate": 0.00023661949196609894, + "loss": 0.5050308609008789, + "mean_token_accuracy": 0.8516230022907257, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.5873846638202668, + "epoch": 2.963226571767497, + "grad_norm": 0.34372401237487793, + "learning_rate": 0.00023327267750758942, + "loss": 0.5120795440673828, + "mean_token_accuracy": 0.8469690150022506, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.4637417015973587, + "eval_loss": 0.5590130090713501, + "eval_mean_token_accuracy": 0.8415542060171651, + "eval_num_tokens": 2005986.0, + "eval_runtime": 69.9647, + "eval_samples_per_second": 19.496, + "eval_steps_per_second": 2.444, + "step": 1266 + }, + { + "entropy": 0.5193097814164981, + "epoch": 3.080664294187426, + "grad_norm": 0.4883718192577362, + "learning_rate": 0.00022975166735111956, + "loss": 0.4416360855102539, + "mean_token_accuracy": 0.8627295740927109, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.4912916052341461, + "epoch": 3.199288256227758, + "grad_norm": 0.42061012983322144, + "learning_rate": 0.00022606248341977406, + "loss": 0.41719051361083986, + "mean_token_accuracy": 0.8705551433563232, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.49565069168806075, + "epoch": 3.31791221826809, + "grad_norm": 0.4688358008861542, + "learning_rate": 0.00022221143526135096, + "loss": 0.425596809387207, + "mean_token_accuracy": 0.8679776644706726, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5045917698740959, + "epoch": 3.4365361803084222, + "grad_norm": 0.4515043795108795, + "learning_rate": 0.0002182051092572507, + "loss": 0.432201042175293, + "mean_token_accuracy": 0.867100123167038, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5014845252037048, + "epoch": 3.5551601423487544, + "grad_norm": 0.5356592535972595, + "learning_rate": 0.00021405035735790138, + "loss": 0.43230159759521486, + "mean_token_accuracy": 0.868084819316864, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.5087948343157769, + "epoch": 3.6737841043890866, + "grad_norm": 0.5144466757774353, + "learning_rate": 0.00020975428536398563, + "loss": 0.4351469039916992, + "mean_token_accuracy": 0.8663349187374115, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5100917038321495, + "epoch": 3.792408066429419, + "grad_norm": 0.39870721101760864, + "learning_rate": 0.00020532424077351232, + "loss": 0.4383781051635742, + "mean_token_accuracy": 0.8638313174247741, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.5078894594311714, + "epoch": 3.9110320284697506, + "grad_norm": 0.5162035226821899, + "learning_rate": 0.00020076780021551676, + "loss": 0.43954910278320314, + "mean_token_accuracy": 0.8654858589172363, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.42526503421409784, + "eval_loss": 0.5683684349060059, + "eval_mean_token_accuracy": 0.8431169801985311, + "eval_num_tokens": 2674648.0, + "eval_runtime": 69.9381, + "eval_samples_per_second": 19.503, + "eval_steps_per_second": 2.445, + "step": 1688 + }, + { + "entropy": 0.4888700656216554, + "epoch": 4.02846975088968, + "grad_norm": 0.5562400817871094, + "learning_rate": 0.00019609275649188284, + "loss": 0.4163688659667969, + "mean_token_accuracy": 0.8697231995939004, + "num_tokens": 2692377.0, + "step": 1700 + }, + { + "entropy": 0.40878518044948575, + "epoch": 4.147093712930012, + "grad_norm": 0.5891883373260498, + "learning_rate": 0.00019130710524944755, + "loss": 0.3318890380859375, + "mean_token_accuracy": 0.8924670344591141, + "num_tokens": 2772306.0, + "step": 1750 + }, + { + "entropy": 0.4020394250750542, + "epoch": 4.265717674970344, + "grad_norm": 0.46362537145614624, + "learning_rate": 0.00018641903130518436, + "loss": 0.3315983963012695, + "mean_token_accuracy": 0.8924845308065414, + "num_tokens": 2851944.0, + "step": 1800 + }, + { + "entropy": 0.4139324098825455, + "epoch": 4.384341637010676, + "grad_norm": 0.419796347618103, + "learning_rate": 0.00018143689464785115, + "loss": 0.34077312469482424, + "mean_token_accuracy": 0.8891813969612121, + "num_tokens": 2932030.0, + "step": 1850 + }, + { + "entropy": 0.4248881351947784, + "epoch": 4.5029655990510085, + "grad_norm": 0.4145258665084839, + "learning_rate": 0.00017636921614004537, + "loss": 0.34736080169677735, + "mean_token_accuracy": 0.8870598292350769, + "num_tokens": 3011165.0, + "step": 1900 + }, + { + "entropy": 0.4218931317329407, + "epoch": 4.621589561091341, + "grad_norm": 0.8228546380996704, + "learning_rate": 0.0001712246629451195, + "loss": 0.3468379211425781, + "mean_token_accuracy": 0.8883900022506714, + "num_tokens": 3091821.0, + "step": 1950 + }, + { + "entropy": 0.40982368469238284, + "epoch": 4.740213523131673, + "grad_norm": 0.523281991481781, + "learning_rate": 0.0001660120337038806, + "loss": 0.3416014862060547, + "mean_token_accuracy": 0.8899656707048416, + "num_tokens": 3169927.0, + "step": 2000 + }, + { + "entropy": 0.41580425947904587, + "epoch": 4.858837485172005, + "grad_norm": 0.5810345411300659, + "learning_rate": 0.0001607402434864264, + "loss": 0.3404109954833984, + "mean_token_accuracy": 0.8906227999925613, + "num_tokens": 3248550.0, + "step": 2050 + }, + { + "entropy": 0.4203647238016128, + "epoch": 4.977461447212336, + "grad_norm": 0.4648055136203766, + "learning_rate": 0.00015541830854485418, + "loss": 0.35020133972167966, + "mean_token_accuracy": 0.8869055426120758, + "num_tokens": 3328838.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3646522021433066, + "eval_loss": 0.6093080639839172, + "eval_mean_token_accuracy": 0.843100773312195, + "eval_num_tokens": 3343310.0, + "eval_runtime": 69.8916, + "eval_samples_per_second": 19.516, + "eval_steps_per_second": 2.447, + "step": 2110 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5494457212504064e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fe15fc4684cbc8b0f9906f045294ddc5e8d46063 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.07029303730572957, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2532/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..93db0a9b479c8a43d27d4e17957dd2e8e6a7a731 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: transformers +model_name: Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3-4B-Base](https://huggingface.co/Qwen/Qwen3-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/lf6jvt2o) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f29670dcf0728e24bf8483b438825be8bcfe1af3 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1266/trainer_state.json @@ -0,0 +1,317 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1266, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.291660257930035e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2d316ca8db36d9e30c996a2eb34976bf38f27543 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-1688/trainer_state.json @@ -0,0 +1,408 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1688, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + }, + { + "entropy": 0.5605672841722315, + "epoch": 3.080664294187426, + "grad_norm": 0.7396782636642456, + "learning_rate": 4.581218014006071e-05, + "loss": 0.49392391204833985, + "mean_token_accuracy": 0.8526303358752318, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.5381742560863495, + "epoch": 3.199288256227758, + "grad_norm": 0.6144187450408936, + "learning_rate": 4.5076561718740064e-05, + "loss": 0.4746086120605469, + "mean_token_accuracy": 0.8577722781896591, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.5408391493558884, + "epoch": 3.31791221826809, + "grad_norm": 0.7017843127250671, + "learning_rate": 4.4308667783536946e-05, + "loss": 0.47829113006591795, + "mean_token_accuracy": 0.8566669344902038, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5474000856280327, + "epoch": 3.4365361803084222, + "grad_norm": 0.8140468597412109, + "learning_rate": 4.350981165023561e-05, + "loss": 0.48412700653076174, + "mean_token_accuracy": 0.8574644386768341, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5438253137469292, + "epoch": 3.5551601423487544, + "grad_norm": 0.7523708343505859, + "learning_rate": 4.268135958873494e-05, + "loss": 0.4800748062133789, + "mean_token_accuracy": 0.8587796002626419, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.550279560983181, + "epoch": 3.6737841043890866, + "grad_norm": 0.693554162979126, + "learning_rate": 4.182472848634056e-05, + "loss": 0.48448719024658204, + "mean_token_accuracy": 0.8568146306276322, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5500899118185043, + "epoch": 3.792408066429419, + "grad_norm": 0.663986086845398, + "learning_rate": 4.0941383424486856e-05, + "loss": 0.48384044647216795, + "mean_token_accuracy": 0.8548811787366867, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.54236083060503, + "epoch": 3.9110320284697506, + "grad_norm": 0.8028485774993896, + "learning_rate": 4.003283517303342e-05, + "loss": 0.4817659378051758, + "mean_token_accuracy": 0.8557876515388488, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45895950382913064, + "eval_loss": 0.5682657957077026, + "eval_mean_token_accuracy": 0.8392008675469292, + "eval_num_tokens": 2674648.0, + "eval_runtime": 67.4386, + "eval_samples_per_second": 20.211, + "eval_steps_per_second": 2.536, + "step": 1688 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2402310441946726e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6f0e68be98325c20d139b4a2b86a5eeb92ab39ee --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2110/trainer_state.json @@ -0,0 +1,509 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2110, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + }, + { + "entropy": 0.5605672841722315, + "epoch": 3.080664294187426, + "grad_norm": 0.7396782636642456, + "learning_rate": 4.581218014006071e-05, + "loss": 0.49392391204833985, + "mean_token_accuracy": 0.8526303358752318, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.5381742560863495, + "epoch": 3.199288256227758, + "grad_norm": 0.6144187450408936, + "learning_rate": 4.5076561718740064e-05, + "loss": 0.4746086120605469, + "mean_token_accuracy": 0.8577722781896591, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.5408391493558884, + "epoch": 3.31791221826809, + "grad_norm": 0.7017843127250671, + "learning_rate": 4.4308667783536946e-05, + "loss": 0.47829113006591795, + "mean_token_accuracy": 0.8566669344902038, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5474000856280327, + "epoch": 3.4365361803084222, + "grad_norm": 0.8140468597412109, + "learning_rate": 4.350981165023561e-05, + "loss": 0.48412700653076174, + "mean_token_accuracy": 0.8574644386768341, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5438253137469292, + "epoch": 3.5551601423487544, + "grad_norm": 0.7523708343505859, + "learning_rate": 4.268135958873494e-05, + "loss": 0.4800748062133789, + "mean_token_accuracy": 0.8587796002626419, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.550279560983181, + "epoch": 3.6737841043890866, + "grad_norm": 0.693554162979126, + "learning_rate": 4.182472848634056e-05, + "loss": 0.48448719024658204, + "mean_token_accuracy": 0.8568146306276322, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5500899118185043, + "epoch": 3.792408066429419, + "grad_norm": 0.663986086845398, + "learning_rate": 4.0941383424486856e-05, + "loss": 0.48384044647216795, + "mean_token_accuracy": 0.8548811787366867, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.54236083060503, + "epoch": 3.9110320284697506, + "grad_norm": 0.8028485774993896, + "learning_rate": 4.003283517303342e-05, + "loss": 0.4817659378051758, + "mean_token_accuracy": 0.8557876515388488, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45895950382913064, + "eval_loss": 0.5682657957077026, + "eval_mean_token_accuracy": 0.8392008675469292, + "eval_num_tokens": 2674648.0, + "eval_runtime": 67.4386, + "eval_samples_per_second": 20.211, + "eval_steps_per_second": 2.536, + "step": 1688 + }, + { + "entropy": 0.5392520301269762, + "epoch": 4.02846975088968, + "grad_norm": 0.7537441849708557, + "learning_rate": 3.9100637606421364e-05, + "loss": 0.47076080322265623, + "mean_token_accuracy": 0.8578027938351487, + "num_tokens": 2692377.0, + "step": 1700 + }, + { + "entropy": 0.4877688565850258, + "epoch": 4.147093712930012, + "grad_norm": 0.8088716864585876, + "learning_rate": 3.814638504610852e-05, + "loss": 0.414734992980957, + "mean_token_accuracy": 0.8727810794115066, + "num_tokens": 2772306.0, + "step": 1750 + }, + { + "entropy": 0.47820613652467725, + "epoch": 4.265717674970344, + "grad_norm": 0.7463680505752563, + "learning_rate": 3.7171709533829017e-05, + "loss": 0.41046337127685545, + "mean_token_accuracy": 0.8740614771842956, + "num_tokens": 2851944.0, + "step": 1800 + }, + { + "entropy": 0.48805462688207624, + "epoch": 4.384341637010676, + "grad_norm": 0.7413962483406067, + "learning_rate": 3.617827804034028e-05, + "loss": 0.41806373596191404, + "mean_token_accuracy": 0.8710768383741379, + "num_tokens": 2932030.0, + "step": 1850 + }, + { + "entropy": 0.49493742316961287, + "epoch": 4.5029655990510085, + "grad_norm": 0.648345947265625, + "learning_rate": 3.5167789614431665e-05, + "loss": 0.4231264114379883, + "mean_token_accuracy": 0.8715304219722748, + "num_tokens": 3011165.0, + "step": 1900 + }, + { + "entropy": 0.48624082893133164, + "epoch": 4.621589561091341, + "grad_norm": 0.9829961657524109, + "learning_rate": 3.414197247707055e-05, + "loss": 0.41783756256103516, + "mean_token_accuracy": 0.8720235615968704, + "num_tokens": 3091821.0, + "step": 1950 + }, + { + "entropy": 0.4882233440876007, + "epoch": 4.740213523131673, + "grad_norm": 0.7564496397972107, + "learning_rate": 3.310258106565575e-05, + "loss": 0.4199684524536133, + "mean_token_accuracy": 0.8720525479316712, + "num_tokens": 3169927.0, + "step": 2000 + }, + { + "entropy": 0.4850278961658478, + "epoch": 4.858837485172005, + "grad_norm": 0.9846013188362122, + "learning_rate": 3.2051393033433424e-05, + "loss": 0.41581527709960936, + "mean_token_accuracy": 0.8731251412630081, + "num_tokens": 3248550.0, + "step": 2050 + }, + { + "entropy": 0.49007617592811586, + "epoch": 4.977461447212336, + "grad_norm": 0.9568490386009216, + "learning_rate": 3.099020620920731e-05, + "loss": 0.42266311645507815, + "mean_token_accuracy": 0.8716179597377777, + "num_tokens": 3328838.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4346666379630217, + "eval_loss": 0.5817540287971497, + "eval_mean_token_accuracy": 0.8398688543609708, + "eval_num_tokens": 3343310.0, + "eval_runtime": 67.4228, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.536, + "step": 2110 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5494457212504064e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d4b2df3e9339dd6230fdaf669e655e455689c440 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2532/trainer_state.json @@ -0,0 +1,600 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2532, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + }, + { + "entropy": 0.5605672841722315, + "epoch": 3.080664294187426, + "grad_norm": 0.7396782636642456, + "learning_rate": 4.581218014006071e-05, + "loss": 0.49392391204833985, + "mean_token_accuracy": 0.8526303358752318, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.5381742560863495, + "epoch": 3.199288256227758, + "grad_norm": 0.6144187450408936, + "learning_rate": 4.5076561718740064e-05, + "loss": 0.4746086120605469, + "mean_token_accuracy": 0.8577722781896591, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.5408391493558884, + "epoch": 3.31791221826809, + "grad_norm": 0.7017843127250671, + "learning_rate": 4.4308667783536946e-05, + "loss": 0.47829113006591795, + "mean_token_accuracy": 0.8566669344902038, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5474000856280327, + "epoch": 3.4365361803084222, + "grad_norm": 0.8140468597412109, + "learning_rate": 4.350981165023561e-05, + "loss": 0.48412700653076174, + "mean_token_accuracy": 0.8574644386768341, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5438253137469292, + "epoch": 3.5551601423487544, + "grad_norm": 0.7523708343505859, + "learning_rate": 4.268135958873494e-05, + "loss": 0.4800748062133789, + "mean_token_accuracy": 0.8587796002626419, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.550279560983181, + "epoch": 3.6737841043890866, + "grad_norm": 0.693554162979126, + "learning_rate": 4.182472848634056e-05, + "loss": 0.48448719024658204, + "mean_token_accuracy": 0.8568146306276322, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5500899118185043, + "epoch": 3.792408066429419, + "grad_norm": 0.663986086845398, + "learning_rate": 4.0941383424486856e-05, + "loss": 0.48384044647216795, + "mean_token_accuracy": 0.8548811787366867, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.54236083060503, + "epoch": 3.9110320284697506, + "grad_norm": 0.8028485774993896, + "learning_rate": 4.003283517303342e-05, + "loss": 0.4817659378051758, + "mean_token_accuracy": 0.8557876515388488, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45895950382913064, + "eval_loss": 0.5682657957077026, + "eval_mean_token_accuracy": 0.8392008675469292, + "eval_num_tokens": 2674648.0, + "eval_runtime": 67.4386, + "eval_samples_per_second": 20.211, + "eval_steps_per_second": 2.536, + "step": 1688 + }, + { + "entropy": 0.5392520301269762, + "epoch": 4.02846975088968, + "grad_norm": 0.7537441849708557, + "learning_rate": 3.9100637606421364e-05, + "loss": 0.47076080322265623, + "mean_token_accuracy": 0.8578027938351487, + "num_tokens": 2692377.0, + "step": 1700 + }, + { + "entropy": 0.4877688565850258, + "epoch": 4.147093712930012, + "grad_norm": 0.8088716864585876, + "learning_rate": 3.814638504610852e-05, + "loss": 0.414734992980957, + "mean_token_accuracy": 0.8727810794115066, + "num_tokens": 2772306.0, + "step": 1750 + }, + { + "entropy": 0.47820613652467725, + "epoch": 4.265717674970344, + "grad_norm": 0.7463680505752563, + "learning_rate": 3.7171709533829017e-05, + "loss": 0.41046337127685545, + "mean_token_accuracy": 0.8740614771842956, + "num_tokens": 2851944.0, + "step": 1800 + }, + { + "entropy": 0.48805462688207624, + "epoch": 4.384341637010676, + "grad_norm": 0.7413962483406067, + "learning_rate": 3.617827804034028e-05, + "loss": 0.41806373596191404, + "mean_token_accuracy": 0.8710768383741379, + "num_tokens": 2932030.0, + "step": 1850 + }, + { + "entropy": 0.49493742316961287, + "epoch": 4.5029655990510085, + "grad_norm": 0.648345947265625, + "learning_rate": 3.5167789614431665e-05, + "loss": 0.4231264114379883, + "mean_token_accuracy": 0.8715304219722748, + "num_tokens": 3011165.0, + "step": 1900 + }, + { + "entropy": 0.48624082893133164, + "epoch": 4.621589561091341, + "grad_norm": 0.9829961657524109, + "learning_rate": 3.414197247707055e-05, + "loss": 0.41783756256103516, + "mean_token_accuracy": 0.8720235615968704, + "num_tokens": 3091821.0, + "step": 1950 + }, + { + "entropy": 0.4882233440876007, + "epoch": 4.740213523131673, + "grad_norm": 0.7564496397972107, + "learning_rate": 3.310258106565575e-05, + "loss": 0.4199684524536133, + "mean_token_accuracy": 0.8720525479316712, + "num_tokens": 3169927.0, + "step": 2000 + }, + { + "entropy": 0.4850278961658478, + "epoch": 4.858837485172005, + "grad_norm": 0.9846013188362122, + "learning_rate": 3.2051393033433424e-05, + "loss": 0.41581527709960936, + "mean_token_accuracy": 0.8731251412630081, + "num_tokens": 3248550.0, + "step": 2050 + }, + { + "entropy": 0.49007617592811586, + "epoch": 4.977461447212336, + "grad_norm": 0.9568490386009216, + "learning_rate": 3.099020620920731e-05, + "loss": 0.42266311645507815, + "mean_token_accuracy": 0.8716179597377777, + "num_tokens": 3328838.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4346666379630217, + "eval_loss": 0.5817540287971497, + "eval_mean_token_accuracy": 0.8398688543609708, + "eval_num_tokens": 3343310.0, + "eval_runtime": 67.4228, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.536, + "step": 2110 + }, + { + "entropy": 0.44751178706535183, + "epoch": 5.094899169632265, + "grad_norm": 0.8798995614051819, + "learning_rate": 2.9920835522543054e-05, + "loss": 0.3644564437866211, + "mean_token_accuracy": 0.8864451273523196, + "num_tokens": 3406471.0, + "step": 2150 + }, + { + "entropy": 0.4300405251979828, + "epoch": 5.213523131672598, + "grad_norm": 0.9511876106262207, + "learning_rate": 2.8845109899725434e-05, + "loss": 0.35093097686767577, + "mean_token_accuracy": 0.8902131456136704, + "num_tokens": 3484267.0, + "step": 2200 + }, + { + "entropy": 0.4232933124899864, + "epoch": 5.33214709371293, + "grad_norm": 0.9091081619262695, + "learning_rate": 2.776486913577716e-05, + "loss": 0.35009231567382815, + "mean_token_accuracy": 0.8907366544008255, + "num_tokens": 3562108.0, + "step": 2250 + }, + { + "entropy": 0.42291093826293946, + "epoch": 5.450771055753262, + "grad_norm": 0.9557301998138428, + "learning_rate": 2.6681960747889137e-05, + "loss": 0.3484030532836914, + "mean_token_accuracy": 0.8915014320611954, + "num_tokens": 3643361.0, + "step": 2300 + }, + { + "entropy": 0.42662834584712983, + "epoch": 5.569395017793594, + "grad_norm": 1.0737801790237427, + "learning_rate": 2.559823681564358e-05, + "loss": 0.3529233551025391, + "mean_token_accuracy": 0.890388081073761, + "num_tokens": 3722789.0, + "step": 2350 + }, + { + "entropy": 0.42317517310380937, + "epoch": 5.6880189798339265, + "grad_norm": 0.9151463508605957, + "learning_rate": 2.4515550813434192e-05, + "loss": 0.34859146118164064, + "mean_token_accuracy": 0.8905287456512451, + "num_tokens": 3801491.0, + "step": 2400 + }, + { + "entropy": 0.4059224596619606, + "epoch": 5.806642941874259, + "grad_norm": 1.039889931678772, + "learning_rate": 2.3435754440500813e-05, + "loss": 0.335848388671875, + "mean_token_accuracy": 0.8958643442392349, + "num_tokens": 3883764.0, + "step": 2450 + }, + { + "entropy": 0.4227695450186729, + "epoch": 5.925266903914591, + "grad_norm": 1.0575917959213257, + "learning_rate": 2.2360694454000018e-05, + "loss": 0.34941902160644533, + "mean_token_accuracy": 0.8892073971033096, + "num_tokens": 3963082.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.39420547523693733, + "eval_loss": 0.6033952832221985, + "eval_mean_token_accuracy": 0.8417927235887762, + "eval_num_tokens": 4011972.0, + "eval_runtime": 67.4533, + "eval_samples_per_second": 20.207, + "eval_steps_per_second": 2.535, + "step": 2532 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8590682591892685e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..becd2c5ddd0b5a5c35134f774e2e03782d95706b --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-2954/trainer_state.json @@ -0,0 +1,701 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2954, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + }, + { + "entropy": 0.5605672841722315, + "epoch": 3.080664294187426, + "grad_norm": 0.7396782636642456, + "learning_rate": 4.581218014006071e-05, + "loss": 0.49392391204833985, + "mean_token_accuracy": 0.8526303358752318, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.5381742560863495, + "epoch": 3.199288256227758, + "grad_norm": 0.6144187450408936, + "learning_rate": 4.5076561718740064e-05, + "loss": 0.4746086120605469, + "mean_token_accuracy": 0.8577722781896591, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.5408391493558884, + "epoch": 3.31791221826809, + "grad_norm": 0.7017843127250671, + "learning_rate": 4.4308667783536946e-05, + "loss": 0.47829113006591795, + "mean_token_accuracy": 0.8566669344902038, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5474000856280327, + "epoch": 3.4365361803084222, + "grad_norm": 0.8140468597412109, + "learning_rate": 4.350981165023561e-05, + "loss": 0.48412700653076174, + "mean_token_accuracy": 0.8574644386768341, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5438253137469292, + "epoch": 3.5551601423487544, + "grad_norm": 0.7523708343505859, + "learning_rate": 4.268135958873494e-05, + "loss": 0.4800748062133789, + "mean_token_accuracy": 0.8587796002626419, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.550279560983181, + "epoch": 3.6737841043890866, + "grad_norm": 0.693554162979126, + "learning_rate": 4.182472848634056e-05, + "loss": 0.48448719024658204, + "mean_token_accuracy": 0.8568146306276322, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5500899118185043, + "epoch": 3.792408066429419, + "grad_norm": 0.663986086845398, + "learning_rate": 4.0941383424486856e-05, + "loss": 0.48384044647216795, + "mean_token_accuracy": 0.8548811787366867, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.54236083060503, + "epoch": 3.9110320284697506, + "grad_norm": 0.8028485774993896, + "learning_rate": 4.003283517303342e-05, + "loss": 0.4817659378051758, + "mean_token_accuracy": 0.8557876515388488, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45895950382913064, + "eval_loss": 0.5682657957077026, + "eval_mean_token_accuracy": 0.8392008675469292, + "eval_num_tokens": 2674648.0, + "eval_runtime": 67.4386, + "eval_samples_per_second": 20.211, + "eval_steps_per_second": 2.536, + "step": 1688 + }, + { + "entropy": 0.5392520301269762, + "epoch": 4.02846975088968, + "grad_norm": 0.7537441849708557, + "learning_rate": 3.9100637606421364e-05, + "loss": 0.47076080322265623, + "mean_token_accuracy": 0.8578027938351487, + "num_tokens": 2692377.0, + "step": 1700 + }, + { + "entropy": 0.4877688565850258, + "epoch": 4.147093712930012, + "grad_norm": 0.8088716864585876, + "learning_rate": 3.814638504610852e-05, + "loss": 0.414734992980957, + "mean_token_accuracy": 0.8727810794115066, + "num_tokens": 2772306.0, + "step": 1750 + }, + { + "entropy": 0.47820613652467725, + "epoch": 4.265717674970344, + "grad_norm": 0.7463680505752563, + "learning_rate": 3.7171709533829017e-05, + "loss": 0.41046337127685545, + "mean_token_accuracy": 0.8740614771842956, + "num_tokens": 2851944.0, + "step": 1800 + }, + { + "entropy": 0.48805462688207624, + "epoch": 4.384341637010676, + "grad_norm": 0.7413962483406067, + "learning_rate": 3.617827804034028e-05, + "loss": 0.41806373596191404, + "mean_token_accuracy": 0.8710768383741379, + "num_tokens": 2932030.0, + "step": 1850 + }, + { + "entropy": 0.49493742316961287, + "epoch": 4.5029655990510085, + "grad_norm": 0.648345947265625, + "learning_rate": 3.5167789614431665e-05, + "loss": 0.4231264114379883, + "mean_token_accuracy": 0.8715304219722748, + "num_tokens": 3011165.0, + "step": 1900 + }, + { + "entropy": 0.48624082893133164, + "epoch": 4.621589561091341, + "grad_norm": 0.9829961657524109, + "learning_rate": 3.414197247707055e-05, + "loss": 0.41783756256103516, + "mean_token_accuracy": 0.8720235615968704, + "num_tokens": 3091821.0, + "step": 1950 + }, + { + "entropy": 0.4882233440876007, + "epoch": 4.740213523131673, + "grad_norm": 0.7564496397972107, + "learning_rate": 3.310258106565575e-05, + "loss": 0.4199684524536133, + "mean_token_accuracy": 0.8720525479316712, + "num_tokens": 3169927.0, + "step": 2000 + }, + { + "entropy": 0.4850278961658478, + "epoch": 4.858837485172005, + "grad_norm": 0.9846013188362122, + "learning_rate": 3.2051393033433424e-05, + "loss": 0.41581527709960936, + "mean_token_accuracy": 0.8731251412630081, + "num_tokens": 3248550.0, + "step": 2050 + }, + { + "entropy": 0.49007617592811586, + "epoch": 4.977461447212336, + "grad_norm": 0.9568490386009216, + "learning_rate": 3.099020620920731e-05, + "loss": 0.42266311645507815, + "mean_token_accuracy": 0.8716179597377777, + "num_tokens": 3328838.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4346666379630217, + "eval_loss": 0.5817540287971497, + "eval_mean_token_accuracy": 0.8398688543609708, + "eval_num_tokens": 3343310.0, + "eval_runtime": 67.4228, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.536, + "step": 2110 + }, + { + "entropy": 0.44751178706535183, + "epoch": 5.094899169632265, + "grad_norm": 0.8798995614051819, + "learning_rate": 2.9920835522543054e-05, + "loss": 0.3644564437866211, + "mean_token_accuracy": 0.8864451273523196, + "num_tokens": 3406471.0, + "step": 2150 + }, + { + "entropy": 0.4300405251979828, + "epoch": 5.213523131672598, + "grad_norm": 0.9511876106262207, + "learning_rate": 2.8845109899725434e-05, + "loss": 0.35093097686767577, + "mean_token_accuracy": 0.8902131456136704, + "num_tokens": 3484267.0, + "step": 2200 + }, + { + "entropy": 0.4232933124899864, + "epoch": 5.33214709371293, + "grad_norm": 0.9091081619262695, + "learning_rate": 2.776486913577716e-05, + "loss": 0.35009231567382815, + "mean_token_accuracy": 0.8907366544008255, + "num_tokens": 3562108.0, + "step": 2250 + }, + { + "entropy": 0.42291093826293946, + "epoch": 5.450771055753262, + "grad_norm": 0.9557301998138428, + "learning_rate": 2.6681960747889137e-05, + "loss": 0.3484030532836914, + "mean_token_accuracy": 0.8915014320611954, + "num_tokens": 3643361.0, + "step": 2300 + }, + { + "entropy": 0.42662834584712983, + "epoch": 5.569395017793594, + "grad_norm": 1.0737801790237427, + "learning_rate": 2.559823681564358e-05, + "loss": 0.3529233551025391, + "mean_token_accuracy": 0.890388081073761, + "num_tokens": 3722789.0, + "step": 2350 + }, + { + "entropy": 0.42317517310380937, + "epoch": 5.6880189798339265, + "grad_norm": 0.9151463508605957, + "learning_rate": 2.4515550813434192e-05, + "loss": 0.34859146118164064, + "mean_token_accuracy": 0.8905287456512451, + "num_tokens": 3801491.0, + "step": 2400 + }, + { + "entropy": 0.4059224596619606, + "epoch": 5.806642941874259, + "grad_norm": 1.039889931678772, + "learning_rate": 2.3435754440500813e-05, + "loss": 0.335848388671875, + "mean_token_accuracy": 0.8958643442392349, + "num_tokens": 3883764.0, + "step": 2450 + }, + { + "entropy": 0.4227695450186729, + "epoch": 5.925266903914591, + "grad_norm": 1.0575917959213257, + "learning_rate": 2.2360694454000018e-05, + "loss": 0.34941902160644533, + "mean_token_accuracy": 0.8892073971033096, + "num_tokens": 3963082.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.39420547523693733, + "eval_loss": 0.6033952832221985, + "eval_mean_token_accuracy": 0.8417927235887762, + "eval_num_tokens": 4011972.0, + "eval_runtime": 67.4533, + "eval_samples_per_second": 20.207, + "eval_steps_per_second": 2.535, + "step": 2532 + }, + { + "entropy": 0.40804383550027407, + "epoch": 6.04270462633452, + "grad_norm": 1.0490775108337402, + "learning_rate": 2.129220951052819e-05, + "loss": 0.3283506774902344, + "mean_token_accuracy": 0.8975986519245186, + "num_tokens": 4040122.0, + "step": 2550 + }, + { + "entropy": 0.3564245703816414, + "epoch": 6.161328588374852, + "grad_norm": 1.0544692277908325, + "learning_rate": 2.0232127021498766e-05, + "loss": 0.26940509796142575, + "mean_token_accuracy": 0.915808516740799, + "num_tokens": 4118433.0, + "step": 2600 + }, + { + "entropy": 0.3520303949713707, + "epoch": 6.279952550415183, + "grad_norm": 1.0909168720245361, + "learning_rate": 1.9182260027751904e-05, + "loss": 0.2703816223144531, + "mean_token_accuracy": 0.9140739214420318, + "num_tokens": 4196965.0, + "step": 2650 + }, + { + "entropy": 0.36691532015800477, + "epoch": 6.398576512455516, + "grad_norm": 1.0912492275238037, + "learning_rate": 1.8144404098741954e-05, + "loss": 0.28067663192749026, + "mean_token_accuracy": 0.9112870472669602, + "num_tokens": 4272810.0, + "step": 2700 + }, + { + "entropy": 0.35482348829507826, + "epoch": 6.517200474495848, + "grad_norm": 1.2286453247070312, + "learning_rate": 1.7120334261605962e-05, + "loss": 0.27153289794921875, + "mean_token_accuracy": 0.9141361153125763, + "num_tokens": 4352908.0, + "step": 2750 + }, + { + "entropy": 0.3551847892999649, + "epoch": 6.63582443653618, + "grad_norm": 1.048604965209961, + "learning_rate": 1.611180196536523e-05, + "loss": 0.26977054595947264, + "mean_token_accuracy": 0.9142266172170639, + "num_tokens": 4431727.0, + "step": 2800 + }, + { + "entropy": 0.34558281034231186, + "epoch": 6.754448398576512, + "grad_norm": 0.9390625953674316, + "learning_rate": 1.51205320854522e-05, + "loss": 0.26851280212402345, + "mean_token_accuracy": 0.9141441309452056, + "num_tokens": 4512844.0, + "step": 2850 + }, + { + "entropy": 0.35204614490270614, + "epoch": 6.8730723606168445, + "grad_norm": 0.9959868788719177, + "learning_rate": 1.4148219973685769e-05, + "loss": 0.2707656478881836, + "mean_token_accuracy": 0.9134996753931045, + "num_tokens": 4594732.0, + "step": 2900 + }, + { + "entropy": 0.3469261267781258, + "epoch": 6.991696322657177, + "grad_norm": 0.9221636652946472, + "learning_rate": 1.3196528558740103e-05, + "loss": 0.269027099609375, + "mean_token_accuracy": 0.9142447954416275, + "num_tokens": 4674745.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.33269924674814905, + "eval_loss": 0.7036522626876831, + "eval_mean_token_accuracy": 0.8354723962426883, + "eval_num_tokens": 4680634.0, + "eval_runtime": 67.4408, + "eval_samples_per_second": 20.21, + "eval_steps_per_second": 2.536, + "step": 2954 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1695579692351488e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7b8b732c87e797ac16cfe21b9517e1d9cf8ac6dc --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3376/trainer_state.json @@ -0,0 +1,792 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3376, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + }, + { + "entropy": 0.5605672841722315, + "epoch": 3.080664294187426, + "grad_norm": 0.7396782636642456, + "learning_rate": 4.581218014006071e-05, + "loss": 0.49392391204833985, + "mean_token_accuracy": 0.8526303358752318, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.5381742560863495, + "epoch": 3.199288256227758, + "grad_norm": 0.6144187450408936, + "learning_rate": 4.5076561718740064e-05, + "loss": 0.4746086120605469, + "mean_token_accuracy": 0.8577722781896591, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.5408391493558884, + "epoch": 3.31791221826809, + "grad_norm": 0.7017843127250671, + "learning_rate": 4.4308667783536946e-05, + "loss": 0.47829113006591795, + "mean_token_accuracy": 0.8566669344902038, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5474000856280327, + "epoch": 3.4365361803084222, + "grad_norm": 0.8140468597412109, + "learning_rate": 4.350981165023561e-05, + "loss": 0.48412700653076174, + "mean_token_accuracy": 0.8574644386768341, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5438253137469292, + "epoch": 3.5551601423487544, + "grad_norm": 0.7523708343505859, + "learning_rate": 4.268135958873494e-05, + "loss": 0.4800748062133789, + "mean_token_accuracy": 0.8587796002626419, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.550279560983181, + "epoch": 3.6737841043890866, + "grad_norm": 0.693554162979126, + "learning_rate": 4.182472848634056e-05, + "loss": 0.48448719024658204, + "mean_token_accuracy": 0.8568146306276322, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5500899118185043, + "epoch": 3.792408066429419, + "grad_norm": 0.663986086845398, + "learning_rate": 4.0941383424486856e-05, + "loss": 0.48384044647216795, + "mean_token_accuracy": 0.8548811787366867, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.54236083060503, + "epoch": 3.9110320284697506, + "grad_norm": 0.8028485774993896, + "learning_rate": 4.003283517303342e-05, + "loss": 0.4817659378051758, + "mean_token_accuracy": 0.8557876515388488, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45895950382913064, + "eval_loss": 0.5682657957077026, + "eval_mean_token_accuracy": 0.8392008675469292, + "eval_num_tokens": 2674648.0, + "eval_runtime": 67.4386, + "eval_samples_per_second": 20.211, + "eval_steps_per_second": 2.536, + "step": 1688 + }, + { + "entropy": 0.5392520301269762, + "epoch": 4.02846975088968, + "grad_norm": 0.7537441849708557, + "learning_rate": 3.9100637606421364e-05, + "loss": 0.47076080322265623, + "mean_token_accuracy": 0.8578027938351487, + "num_tokens": 2692377.0, + "step": 1700 + }, + { + "entropy": 0.4877688565850258, + "epoch": 4.147093712930012, + "grad_norm": 0.8088716864585876, + "learning_rate": 3.814638504610852e-05, + "loss": 0.414734992980957, + "mean_token_accuracy": 0.8727810794115066, + "num_tokens": 2772306.0, + "step": 1750 + }, + { + "entropy": 0.47820613652467725, + "epoch": 4.265717674970344, + "grad_norm": 0.7463680505752563, + "learning_rate": 3.7171709533829017e-05, + "loss": 0.41046337127685545, + "mean_token_accuracy": 0.8740614771842956, + "num_tokens": 2851944.0, + "step": 1800 + }, + { + "entropy": 0.48805462688207624, + "epoch": 4.384341637010676, + "grad_norm": 0.7413962483406067, + "learning_rate": 3.617827804034028e-05, + "loss": 0.41806373596191404, + "mean_token_accuracy": 0.8710768383741379, + "num_tokens": 2932030.0, + "step": 1850 + }, + { + "entropy": 0.49493742316961287, + "epoch": 4.5029655990510085, + "grad_norm": 0.648345947265625, + "learning_rate": 3.5167789614431665e-05, + "loss": 0.4231264114379883, + "mean_token_accuracy": 0.8715304219722748, + "num_tokens": 3011165.0, + "step": 1900 + }, + { + "entropy": 0.48624082893133164, + "epoch": 4.621589561091341, + "grad_norm": 0.9829961657524109, + "learning_rate": 3.414197247707055e-05, + "loss": 0.41783756256103516, + "mean_token_accuracy": 0.8720235615968704, + "num_tokens": 3091821.0, + "step": 1950 + }, + { + "entropy": 0.4882233440876007, + "epoch": 4.740213523131673, + "grad_norm": 0.7564496397972107, + "learning_rate": 3.310258106565575e-05, + "loss": 0.4199684524536133, + "mean_token_accuracy": 0.8720525479316712, + "num_tokens": 3169927.0, + "step": 2000 + }, + { + "entropy": 0.4850278961658478, + "epoch": 4.858837485172005, + "grad_norm": 0.9846013188362122, + "learning_rate": 3.2051393033433424e-05, + "loss": 0.41581527709960936, + "mean_token_accuracy": 0.8731251412630081, + "num_tokens": 3248550.0, + "step": 2050 + }, + { + "entropy": 0.49007617592811586, + "epoch": 4.977461447212336, + "grad_norm": 0.9568490386009216, + "learning_rate": 3.099020620920731e-05, + "loss": 0.42266311645507815, + "mean_token_accuracy": 0.8716179597377777, + "num_tokens": 3328838.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4346666379630217, + "eval_loss": 0.5817540287971497, + "eval_mean_token_accuracy": 0.8398688543609708, + "eval_num_tokens": 3343310.0, + "eval_runtime": 67.4228, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.536, + "step": 2110 + }, + { + "entropy": 0.44751178706535183, + "epoch": 5.094899169632265, + "grad_norm": 0.8798995614051819, + "learning_rate": 2.9920835522543054e-05, + "loss": 0.3644564437866211, + "mean_token_accuracy": 0.8864451273523196, + "num_tokens": 3406471.0, + "step": 2150 + }, + { + "entropy": 0.4300405251979828, + "epoch": 5.213523131672598, + "grad_norm": 0.9511876106262207, + "learning_rate": 2.8845109899725434e-05, + "loss": 0.35093097686767577, + "mean_token_accuracy": 0.8902131456136704, + "num_tokens": 3484267.0, + "step": 2200 + }, + { + "entropy": 0.4232933124899864, + "epoch": 5.33214709371293, + "grad_norm": 0.9091081619262695, + "learning_rate": 2.776486913577716e-05, + "loss": 0.35009231567382815, + "mean_token_accuracy": 0.8907366544008255, + "num_tokens": 3562108.0, + "step": 2250 + }, + { + "entropy": 0.42291093826293946, + "epoch": 5.450771055753262, + "grad_norm": 0.9557301998138428, + "learning_rate": 2.6681960747889137e-05, + "loss": 0.3484030532836914, + "mean_token_accuracy": 0.8915014320611954, + "num_tokens": 3643361.0, + "step": 2300 + }, + { + "entropy": 0.42662834584712983, + "epoch": 5.569395017793594, + "grad_norm": 1.0737801790237427, + "learning_rate": 2.559823681564358e-05, + "loss": 0.3529233551025391, + "mean_token_accuracy": 0.890388081073761, + "num_tokens": 3722789.0, + "step": 2350 + }, + { + "entropy": 0.42317517310380937, + "epoch": 5.6880189798339265, + "grad_norm": 0.9151463508605957, + "learning_rate": 2.4515550813434192e-05, + "loss": 0.34859146118164064, + "mean_token_accuracy": 0.8905287456512451, + "num_tokens": 3801491.0, + "step": 2400 + }, + { + "entropy": 0.4059224596619606, + "epoch": 5.806642941874259, + "grad_norm": 1.039889931678772, + "learning_rate": 2.3435754440500813e-05, + "loss": 0.335848388671875, + "mean_token_accuracy": 0.8958643442392349, + "num_tokens": 3883764.0, + "step": 2450 + }, + { + "entropy": 0.4227695450186729, + "epoch": 5.925266903914591, + "grad_norm": 1.0575917959213257, + "learning_rate": 2.2360694454000018e-05, + "loss": 0.34941902160644533, + "mean_token_accuracy": 0.8892073971033096, + "num_tokens": 3963082.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.39420547523693733, + "eval_loss": 0.6033952832221985, + "eval_mean_token_accuracy": 0.8417927235887762, + "eval_num_tokens": 4011972.0, + "eval_runtime": 67.4533, + "eval_samples_per_second": 20.207, + "eval_steps_per_second": 2.535, + "step": 2532 + }, + { + "entropy": 0.40804383550027407, + "epoch": 6.04270462633452, + "grad_norm": 1.0490775108337402, + "learning_rate": 2.129220951052819e-05, + "loss": 0.3283506774902344, + "mean_token_accuracy": 0.8975986519245186, + "num_tokens": 4040122.0, + "step": 2550 + }, + { + "entropy": 0.3564245703816414, + "epoch": 6.161328588374852, + "grad_norm": 1.0544692277908325, + "learning_rate": 2.0232127021498766e-05, + "loss": 0.26940509796142575, + "mean_token_accuracy": 0.915808516740799, + "num_tokens": 4118433.0, + "step": 2600 + }, + { + "entropy": 0.3520303949713707, + "epoch": 6.279952550415183, + "grad_norm": 1.0909168720245361, + "learning_rate": 1.9182260027751904e-05, + "loss": 0.2703816223144531, + "mean_token_accuracy": 0.9140739214420318, + "num_tokens": 4196965.0, + "step": 2650 + }, + { + "entropy": 0.36691532015800477, + "epoch": 6.398576512455516, + "grad_norm": 1.0912492275238037, + "learning_rate": 1.8144404098741954e-05, + "loss": 0.28067663192749026, + "mean_token_accuracy": 0.9112870472669602, + "num_tokens": 4272810.0, + "step": 2700 + }, + { + "entropy": 0.35482348829507826, + "epoch": 6.517200474495848, + "grad_norm": 1.2286453247070312, + "learning_rate": 1.7120334261605962e-05, + "loss": 0.27153289794921875, + "mean_token_accuracy": 0.9141361153125763, + "num_tokens": 4352908.0, + "step": 2750 + }, + { + "entropy": 0.3551847892999649, + "epoch": 6.63582443653618, + "grad_norm": 1.048604965209961, + "learning_rate": 1.611180196536523e-05, + "loss": 0.26977054595947264, + "mean_token_accuracy": 0.9142266172170639, + "num_tokens": 4431727.0, + "step": 2800 + }, + { + "entropy": 0.34558281034231186, + "epoch": 6.754448398576512, + "grad_norm": 0.9390625953674316, + "learning_rate": 1.51205320854522e-05, + "loss": 0.26851280212402345, + "mean_token_accuracy": 0.9141441309452056, + "num_tokens": 4512844.0, + "step": 2850 + }, + { + "entropy": 0.35204614490270614, + "epoch": 6.8730723606168445, + "grad_norm": 0.9959868788719177, + "learning_rate": 1.4148219973685769e-05, + "loss": 0.2707656478881836, + "mean_token_accuracy": 0.9134996753931045, + "num_tokens": 4594732.0, + "step": 2900 + }, + { + "entropy": 0.3469261267781258, + "epoch": 6.991696322657177, + "grad_norm": 0.9221636652946472, + "learning_rate": 1.3196528558740103e-05, + "loss": 0.269027099609375, + "mean_token_accuracy": 0.9142447954416275, + "num_tokens": 4674745.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.33269924674814905, + "eval_loss": 0.7036522626876831, + "eval_mean_token_accuracy": 0.8354723962426883, + "eval_num_tokens": 4680634.0, + "eval_runtime": 67.4408, + "eval_samples_per_second": 20.21, + "eval_steps_per_second": 2.536, + "step": 2954 + }, + { + "entropy": 0.3100580937332577, + "epoch": 7.109134045077106, + "grad_norm": 1.0155000686645508, + "learning_rate": 1.2267085502066457e-05, + "loss": 0.20666887283325194, + "mean_token_accuracy": 0.9355484825192075, + "num_tokens": 4751686.0, + "step": 3000 + }, + { + "entropy": 0.28997830346226694, + "epoch": 7.227758007117438, + "grad_norm": 1.0495970249176025, + "learning_rate": 1.1361480414131762e-05, + "loss": 0.19952856063842772, + "mean_token_accuracy": 0.9384147107601166, + "num_tokens": 4832492.0, + "step": 3050 + }, + { + "entropy": 0.30070391461253165, + "epoch": 7.34638196915777, + "grad_norm": 1.3204537630081177, + "learning_rate": 1.0481262135735177e-05, + "loss": 0.2062089729309082, + "mean_token_accuracy": 0.936167967915535, + "num_tokens": 4910805.0, + "step": 3100 + }, + { + "entropy": 0.289249883890152, + "epoch": 7.465005931198102, + "grad_norm": 1.131684422492981, + "learning_rate": 9.627936089052345e-06, + "loss": 0.19861122131347655, + "mean_token_accuracy": 0.9381200420856476, + "num_tokens": 4992483.0, + "step": 3150 + }, + { + "entropy": 0.2961649990081787, + "epoch": 7.5836298932384345, + "grad_norm": 1.060278296470642, + "learning_rate": 8.802961702937752e-06, + "loss": 0.2034711265563965, + "mean_token_accuracy": 0.9352211368083954, + "num_tokens": 5071712.0, + "step": 3200 + }, + { + "entropy": 0.2959682309627533, + "epoch": 7.702253855278766, + "grad_norm": 1.09508216381073, + "learning_rate": 8.007749916888647e-06, + "loss": 0.20609720230102538, + "mean_token_accuracy": 0.935073122382164, + "num_tokens": 5149157.0, + "step": 3250 + }, + { + "entropy": 0.2906328068673611, + "epoch": 7.820877817319099, + "grad_norm": 0.9429389834403992, + "learning_rate": 7.243660767939519e-06, + "loss": 0.19851903915405272, + "mean_token_accuracy": 0.9371049195528031, + "num_tokens": 5230144.0, + "step": 3300 + }, + { + "entropy": 0.2917777775228024, + "epoch": 7.93950177935943, + "grad_norm": 0.9275627136230469, + "learning_rate": 6.512001064614148e-06, + "loss": 0.20342073440551758, + "mean_token_accuracy": 0.9363774961233139, + "num_tokens": 5309214.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.29100701056028666, + "eval_loss": 0.8126235008239746, + "eval_mean_token_accuracy": 0.8302465771373949, + "eval_num_tokens": 5349296.0, + "eval_runtime": 67.4144, + "eval_samples_per_second": 20.218, + "eval_steps_per_second": 2.537, + "step": 3376 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.4790822837961933e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..90245680c056acde7b176c88d19a56f227e2412c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-3798/trainer_state.json @@ -0,0 +1,883 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3798, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + }, + { + "entropy": 0.5605672841722315, + "epoch": 3.080664294187426, + "grad_norm": 0.7396782636642456, + "learning_rate": 4.581218014006071e-05, + "loss": 0.49392391204833985, + "mean_token_accuracy": 0.8526303358752318, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.5381742560863495, + "epoch": 3.199288256227758, + "grad_norm": 0.6144187450408936, + "learning_rate": 4.5076561718740064e-05, + "loss": 0.4746086120605469, + "mean_token_accuracy": 0.8577722781896591, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.5408391493558884, + "epoch": 3.31791221826809, + "grad_norm": 0.7017843127250671, + "learning_rate": 4.4308667783536946e-05, + "loss": 0.47829113006591795, + "mean_token_accuracy": 0.8566669344902038, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5474000856280327, + "epoch": 3.4365361803084222, + "grad_norm": 0.8140468597412109, + "learning_rate": 4.350981165023561e-05, + "loss": 0.48412700653076174, + "mean_token_accuracy": 0.8574644386768341, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5438253137469292, + "epoch": 3.5551601423487544, + "grad_norm": 0.7523708343505859, + "learning_rate": 4.268135958873494e-05, + "loss": 0.4800748062133789, + "mean_token_accuracy": 0.8587796002626419, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.550279560983181, + "epoch": 3.6737841043890866, + "grad_norm": 0.693554162979126, + "learning_rate": 4.182472848634056e-05, + "loss": 0.48448719024658204, + "mean_token_accuracy": 0.8568146306276322, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5500899118185043, + "epoch": 3.792408066429419, + "grad_norm": 0.663986086845398, + "learning_rate": 4.0941383424486856e-05, + "loss": 0.48384044647216795, + "mean_token_accuracy": 0.8548811787366867, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.54236083060503, + "epoch": 3.9110320284697506, + "grad_norm": 0.8028485774993896, + "learning_rate": 4.003283517303342e-05, + "loss": 0.4817659378051758, + "mean_token_accuracy": 0.8557876515388488, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45895950382913064, + "eval_loss": 0.5682657957077026, + "eval_mean_token_accuracy": 0.8392008675469292, + "eval_num_tokens": 2674648.0, + "eval_runtime": 67.4386, + "eval_samples_per_second": 20.211, + "eval_steps_per_second": 2.536, + "step": 1688 + }, + { + "entropy": 0.5392520301269762, + "epoch": 4.02846975088968, + "grad_norm": 0.7537441849708557, + "learning_rate": 3.9100637606421364e-05, + "loss": 0.47076080322265623, + "mean_token_accuracy": 0.8578027938351487, + "num_tokens": 2692377.0, + "step": 1700 + }, + { + "entropy": 0.4877688565850258, + "epoch": 4.147093712930012, + "grad_norm": 0.8088716864585876, + "learning_rate": 3.814638504610852e-05, + "loss": 0.414734992980957, + "mean_token_accuracy": 0.8727810794115066, + "num_tokens": 2772306.0, + "step": 1750 + }, + { + "entropy": 0.47820613652467725, + "epoch": 4.265717674970344, + "grad_norm": 0.7463680505752563, + "learning_rate": 3.7171709533829017e-05, + "loss": 0.41046337127685545, + "mean_token_accuracy": 0.8740614771842956, + "num_tokens": 2851944.0, + "step": 1800 + }, + { + "entropy": 0.48805462688207624, + "epoch": 4.384341637010676, + "grad_norm": 0.7413962483406067, + "learning_rate": 3.617827804034028e-05, + "loss": 0.41806373596191404, + "mean_token_accuracy": 0.8710768383741379, + "num_tokens": 2932030.0, + "step": 1850 + }, + { + "entropy": 0.49493742316961287, + "epoch": 4.5029655990510085, + "grad_norm": 0.648345947265625, + "learning_rate": 3.5167789614431665e-05, + "loss": 0.4231264114379883, + "mean_token_accuracy": 0.8715304219722748, + "num_tokens": 3011165.0, + "step": 1900 + }, + { + "entropy": 0.48624082893133164, + "epoch": 4.621589561091341, + "grad_norm": 0.9829961657524109, + "learning_rate": 3.414197247707055e-05, + "loss": 0.41783756256103516, + "mean_token_accuracy": 0.8720235615968704, + "num_tokens": 3091821.0, + "step": 1950 + }, + { + "entropy": 0.4882233440876007, + "epoch": 4.740213523131673, + "grad_norm": 0.7564496397972107, + "learning_rate": 3.310258106565575e-05, + "loss": 0.4199684524536133, + "mean_token_accuracy": 0.8720525479316712, + "num_tokens": 3169927.0, + "step": 2000 + }, + { + "entropy": 0.4850278961658478, + "epoch": 4.858837485172005, + "grad_norm": 0.9846013188362122, + "learning_rate": 3.2051393033433424e-05, + "loss": 0.41581527709960936, + "mean_token_accuracy": 0.8731251412630081, + "num_tokens": 3248550.0, + "step": 2050 + }, + { + "entropy": 0.49007617592811586, + "epoch": 4.977461447212336, + "grad_norm": 0.9568490386009216, + "learning_rate": 3.099020620920731e-05, + "loss": 0.42266311645507815, + "mean_token_accuracy": 0.8716179597377777, + "num_tokens": 3328838.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4346666379630217, + "eval_loss": 0.5817540287971497, + "eval_mean_token_accuracy": 0.8398688543609708, + "eval_num_tokens": 3343310.0, + "eval_runtime": 67.4228, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.536, + "step": 2110 + }, + { + "entropy": 0.44751178706535183, + "epoch": 5.094899169632265, + "grad_norm": 0.8798995614051819, + "learning_rate": 2.9920835522543054e-05, + "loss": 0.3644564437866211, + "mean_token_accuracy": 0.8864451273523196, + "num_tokens": 3406471.0, + "step": 2150 + }, + { + "entropy": 0.4300405251979828, + "epoch": 5.213523131672598, + "grad_norm": 0.9511876106262207, + "learning_rate": 2.8845109899725434e-05, + "loss": 0.35093097686767577, + "mean_token_accuracy": 0.8902131456136704, + "num_tokens": 3484267.0, + "step": 2200 + }, + { + "entropy": 0.4232933124899864, + "epoch": 5.33214709371293, + "grad_norm": 0.9091081619262695, + "learning_rate": 2.776486913577716e-05, + "loss": 0.35009231567382815, + "mean_token_accuracy": 0.8907366544008255, + "num_tokens": 3562108.0, + "step": 2250 + }, + { + "entropy": 0.42291093826293946, + "epoch": 5.450771055753262, + "grad_norm": 0.9557301998138428, + "learning_rate": 2.6681960747889137e-05, + "loss": 0.3484030532836914, + "mean_token_accuracy": 0.8915014320611954, + "num_tokens": 3643361.0, + "step": 2300 + }, + { + "entropy": 0.42662834584712983, + "epoch": 5.569395017793594, + "grad_norm": 1.0737801790237427, + "learning_rate": 2.559823681564358e-05, + "loss": 0.3529233551025391, + "mean_token_accuracy": 0.890388081073761, + "num_tokens": 3722789.0, + "step": 2350 + }, + { + "entropy": 0.42317517310380937, + "epoch": 5.6880189798339265, + "grad_norm": 0.9151463508605957, + "learning_rate": 2.4515550813434192e-05, + "loss": 0.34859146118164064, + "mean_token_accuracy": 0.8905287456512451, + "num_tokens": 3801491.0, + "step": 2400 + }, + { + "entropy": 0.4059224596619606, + "epoch": 5.806642941874259, + "grad_norm": 1.039889931678772, + "learning_rate": 2.3435754440500813e-05, + "loss": 0.335848388671875, + "mean_token_accuracy": 0.8958643442392349, + "num_tokens": 3883764.0, + "step": 2450 + }, + { + "entropy": 0.4227695450186729, + "epoch": 5.925266903914591, + "grad_norm": 1.0575917959213257, + "learning_rate": 2.2360694454000018e-05, + "loss": 0.34941902160644533, + "mean_token_accuracy": 0.8892073971033096, + "num_tokens": 3963082.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.39420547523693733, + "eval_loss": 0.6033952832221985, + "eval_mean_token_accuracy": 0.8417927235887762, + "eval_num_tokens": 4011972.0, + "eval_runtime": 67.4533, + "eval_samples_per_second": 20.207, + "eval_steps_per_second": 2.535, + "step": 2532 + }, + { + "entropy": 0.40804383550027407, + "epoch": 6.04270462633452, + "grad_norm": 1.0490775108337402, + "learning_rate": 2.129220951052819e-05, + "loss": 0.3283506774902344, + "mean_token_accuracy": 0.8975986519245186, + "num_tokens": 4040122.0, + "step": 2550 + }, + { + "entropy": 0.3564245703816414, + "epoch": 6.161328588374852, + "grad_norm": 1.0544692277908325, + "learning_rate": 2.0232127021498766e-05, + "loss": 0.26940509796142575, + "mean_token_accuracy": 0.915808516740799, + "num_tokens": 4118433.0, + "step": 2600 + }, + { + "entropy": 0.3520303949713707, + "epoch": 6.279952550415183, + "grad_norm": 1.0909168720245361, + "learning_rate": 1.9182260027751904e-05, + "loss": 0.2703816223144531, + "mean_token_accuracy": 0.9140739214420318, + "num_tokens": 4196965.0, + "step": 2650 + }, + { + "entropy": 0.36691532015800477, + "epoch": 6.398576512455516, + "grad_norm": 1.0912492275238037, + "learning_rate": 1.8144404098741954e-05, + "loss": 0.28067663192749026, + "mean_token_accuracy": 0.9112870472669602, + "num_tokens": 4272810.0, + "step": 2700 + }, + { + "entropy": 0.35482348829507826, + "epoch": 6.517200474495848, + "grad_norm": 1.2286453247070312, + "learning_rate": 1.7120334261605962e-05, + "loss": 0.27153289794921875, + "mean_token_accuracy": 0.9141361153125763, + "num_tokens": 4352908.0, + "step": 2750 + }, + { + "entropy": 0.3551847892999649, + "epoch": 6.63582443653618, + "grad_norm": 1.048604965209961, + "learning_rate": 1.611180196536523e-05, + "loss": 0.26977054595947264, + "mean_token_accuracy": 0.9142266172170639, + "num_tokens": 4431727.0, + "step": 2800 + }, + { + "entropy": 0.34558281034231186, + "epoch": 6.754448398576512, + "grad_norm": 0.9390625953674316, + "learning_rate": 1.51205320854522e-05, + "loss": 0.26851280212402345, + "mean_token_accuracy": 0.9141441309452056, + "num_tokens": 4512844.0, + "step": 2850 + }, + { + "entropy": 0.35204614490270614, + "epoch": 6.8730723606168445, + "grad_norm": 0.9959868788719177, + "learning_rate": 1.4148219973685769e-05, + "loss": 0.2707656478881836, + "mean_token_accuracy": 0.9134996753931045, + "num_tokens": 4594732.0, + "step": 2900 + }, + { + "entropy": 0.3469261267781258, + "epoch": 6.991696322657177, + "grad_norm": 0.9221636652946472, + "learning_rate": 1.3196528558740103e-05, + "loss": 0.269027099609375, + "mean_token_accuracy": 0.9142447954416275, + "num_tokens": 4674745.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.33269924674814905, + "eval_loss": 0.7036522626876831, + "eval_mean_token_accuracy": 0.8354723962426883, + "eval_num_tokens": 4680634.0, + "eval_runtime": 67.4408, + "eval_samples_per_second": 20.21, + "eval_steps_per_second": 2.536, + "step": 2954 + }, + { + "entropy": 0.3100580937332577, + "epoch": 7.109134045077106, + "grad_norm": 1.0155000686645508, + "learning_rate": 1.2267085502066457e-05, + "loss": 0.20666887283325194, + "mean_token_accuracy": 0.9355484825192075, + "num_tokens": 4751686.0, + "step": 3000 + }, + { + "entropy": 0.28997830346226694, + "epoch": 7.227758007117438, + "grad_norm": 1.0495970249176025, + "learning_rate": 1.1361480414131762e-05, + "loss": 0.19952856063842772, + "mean_token_accuracy": 0.9384147107601166, + "num_tokens": 4832492.0, + "step": 3050 + }, + { + "entropy": 0.30070391461253165, + "epoch": 7.34638196915777, + "grad_norm": 1.3204537630081177, + "learning_rate": 1.0481262135735177e-05, + "loss": 0.2062089729309082, + "mean_token_accuracy": 0.936167967915535, + "num_tokens": 4910805.0, + "step": 3100 + }, + { + "entropy": 0.289249883890152, + "epoch": 7.465005931198102, + "grad_norm": 1.131684422492981, + "learning_rate": 9.627936089052345e-06, + "loss": 0.19861122131347655, + "mean_token_accuracy": 0.9381200420856476, + "num_tokens": 4992483.0, + "step": 3150 + }, + { + "entropy": 0.2961649990081787, + "epoch": 7.5836298932384345, + "grad_norm": 1.060278296470642, + "learning_rate": 8.802961702937752e-06, + "loss": 0.2034711265563965, + "mean_token_accuracy": 0.9352211368083954, + "num_tokens": 5071712.0, + "step": 3200 + }, + { + "entropy": 0.2959682309627533, + "epoch": 7.702253855278766, + "grad_norm": 1.09508216381073, + "learning_rate": 8.007749916888647e-06, + "loss": 0.20609720230102538, + "mean_token_accuracy": 0.935073122382164, + "num_tokens": 5149157.0, + "step": 3250 + }, + { + "entropy": 0.2906328068673611, + "epoch": 7.820877817319099, + "grad_norm": 0.9429389834403992, + "learning_rate": 7.243660767939519e-06, + "loss": 0.19851903915405272, + "mean_token_accuracy": 0.9371049195528031, + "num_tokens": 5230144.0, + "step": 3300 + }, + { + "entropy": 0.2917777775228024, + "epoch": 7.93950177935943, + "grad_norm": 0.9275627136230469, + "learning_rate": 6.512001064614148e-06, + "loss": 0.20342073440551758, + "mean_token_accuracy": 0.9363774961233139, + "num_tokens": 5309214.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.29100701056028666, + "eval_loss": 0.8126235008239746, + "eval_mean_token_accuracy": 0.8302465771373949, + "eval_num_tokens": 5349296.0, + "eval_runtime": 67.4144, + "eval_samples_per_second": 20.218, + "eval_steps_per_second": 2.537, + "step": 3376 + }, + { + "entropy": 0.27969253860940835, + "epoch": 8.05693950177936, + "grad_norm": 1.0733296871185303, + "learning_rate": 5.814022151913483e-06, + "loss": 0.18230682373046875, + "mean_token_accuracy": 0.9442709967343494, + "num_tokens": 5389623.0, + "step": 3400 + }, + { + "entropy": 0.249246060103178, + "epoch": 8.175563463819692, + "grad_norm": 1.1532745361328125, + "learning_rate": 5.150917771161802e-06, + "loss": 0.15249720573425293, + "mean_token_accuracy": 0.954327083826065, + "num_tokens": 5470361.0, + "step": 3450 + }, + { + "entropy": 0.2619244894385338, + "epoch": 8.294187425860024, + "grad_norm": 0.9613165855407715, + "learning_rate": 4.523822018371486e-06, + "loss": 0.1615569305419922, + "mean_token_accuracy": 0.9513245701789856, + "num_tokens": 5546553.0, + "step": 3500 + }, + { + "entropy": 0.25780971825122834, + "epoch": 8.412811387900355, + "grad_norm": 0.9987137317657471, + "learning_rate": 3.93380740461811e-06, + "loss": 0.16112512588500977, + "mean_token_accuracy": 0.9494028472900391, + "num_tokens": 5623387.0, + "step": 3550 + }, + { + "entropy": 0.25041655868291857, + "epoch": 8.531435349940688, + "grad_norm": 0.9792889356613159, + "learning_rate": 3.381883021743194e-06, + "loss": 0.15699137687683107, + "mean_token_accuracy": 0.9521727949380875, + "num_tokens": 5703112.0, + "step": 3600 + }, + { + "entropy": 0.2552810598909855, + "epoch": 8.65005931198102, + "grad_norm": 1.1667588949203491, + "learning_rate": 2.868992816521833e-06, + "loss": 0.15912227630615233, + "mean_token_accuracy": 0.9508797514438629, + "num_tokens": 5782342.0, + "step": 3650 + }, + { + "entropy": 0.2524907249212265, + "epoch": 8.768683274021353, + "grad_norm": 1.2174683809280396, + "learning_rate": 2.3960139762467253e-06, + "loss": 0.1558846950531006, + "mean_token_accuracy": 0.9524789917469024, + "num_tokens": 5860871.0, + "step": 3700 + }, + { + "entropy": 0.24737386167049408, + "epoch": 8.887307236061684, + "grad_norm": 1.1920585632324219, + "learning_rate": 1.963755428489844e-06, + "loss": 0.15519388198852538, + "mean_token_accuracy": 0.9529236805438995, + "num_tokens": 5941467.0, + "step": 3750 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2624759337706873, + "eval_loss": 0.9007481336593628, + "eval_mean_token_accuracy": 0.8296350861153408, + "eval_num_tokens": 6017958.0, + "eval_runtime": 67.3932, + "eval_samples_per_second": 20.225, + "eval_steps_per_second": 2.537, + "step": 3798 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.789854502985892e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..15932c93161f9ce7b33962bbd06ad14c895f516d --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-422/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 422, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.104410660873421e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1b131e6c5131054f17fe4627ec8f104a10db49df --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-4220/trainer_state.json @@ -0,0 +1,984 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4220, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + }, + { + "entropy": 0.6679320124664692, + "epoch": 2.01423487544484, + "grad_norm": 0.6750771999359131, + "learning_rate": 5.0792585674448424e-05, + "loss": 0.6021934509277344, + "mean_token_accuracy": 0.8299772781555099, + "num_tokens": 1346812.0, + "step": 850 + }, + { + "entropy": 0.6181541180610657, + "epoch": 2.132858837485172, + "grad_norm": 0.6573153734207153, + "learning_rate": 5.03967748552471e-05, + "loss": 0.5524405670166016, + "mean_token_accuracy": 0.841307817697525, + "num_tokens": 1424686.0, + "step": 900 + }, + { + "entropy": 0.6198816159367562, + "epoch": 2.2514827995255042, + "grad_norm": 0.7040820717811584, + "learning_rate": 4.995958945318074e-05, + "loss": 0.5466666793823243, + "mean_token_accuracy": 0.8413545161485672, + "num_tokens": 1502627.0, + "step": 950 + }, + { + "entropy": 0.6106881147623062, + "epoch": 2.3701067615658364, + "grad_norm": 0.6758481860160828, + "learning_rate": 4.948177717891986e-05, + "loss": 0.5471043395996094, + "mean_token_accuracy": 0.842030581831932, + "num_tokens": 1582541.0, + "step": 1000 + }, + { + "entropy": 0.6231731602549553, + "epoch": 2.488730723606168, + "grad_norm": 0.6890319585800171, + "learning_rate": 4.8964155226575416e-05, + "loss": 0.5574079513549804, + "mean_token_accuracy": 0.8374398946762085, + "num_tokens": 1662384.0, + "step": 1050 + }, + { + "entropy": 0.6178388431668281, + "epoch": 2.6073546856465004, + "grad_norm": 0.6180830001831055, + "learning_rate": 4.84076088760658e-05, + "loss": 0.5521801376342773, + "mean_token_accuracy": 0.840655015707016, + "num_tokens": 1740050.0, + "step": 1100 + }, + { + "entropy": 0.6119217535853386, + "epoch": 2.7259786476868326, + "grad_norm": 0.6286294460296631, + "learning_rate": 4.78130899790378e-05, + "loss": 0.5491256713867188, + "mean_token_accuracy": 0.8408385002613068, + "num_tokens": 1820783.0, + "step": 1150 + }, + { + "entropy": 0.591200729906559, + "epoch": 2.844602609727165, + "grad_norm": 0.731385350227356, + "learning_rate": 4.7181615330931134e-05, + "loss": 0.5327339172363281, + "mean_token_accuracy": 0.8457299703359604, + "num_tokens": 1902245.0, + "step": 1200 + }, + { + "entropy": 0.6057731649279594, + "epoch": 2.963226571767497, + "grad_norm": 0.6081883907318115, + "learning_rate": 4.6514264931970685e-05, + "loss": 0.5397920227050781, + "mean_token_accuracy": 0.8411379367113113, + "num_tokens": 1981838.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5040011398973521, + "eval_loss": 0.5577003955841064, + "eval_mean_token_accuracy": 0.8407086039844313, + "eval_num_tokens": 2005986.0, + "eval_runtime": 67.4469, + "eval_samples_per_second": 20.208, + "eval_steps_per_second": 2.535, + "step": 1266 + }, + { + "entropy": 0.5605672841722315, + "epoch": 3.080664294187426, + "grad_norm": 0.7396782636642456, + "learning_rate": 4.581218014006071e-05, + "loss": 0.49392391204833985, + "mean_token_accuracy": 0.8526303358752318, + "num_tokens": 2061302.0, + "step": 1300 + }, + { + "entropy": 0.5381742560863495, + "epoch": 3.199288256227758, + "grad_norm": 0.6144187450408936, + "learning_rate": 4.5076561718740064e-05, + "loss": 0.4746086120605469, + "mean_token_accuracy": 0.8577722781896591, + "num_tokens": 2142124.0, + "step": 1350 + }, + { + "entropy": 0.5408391493558884, + "epoch": 3.31791221826809, + "grad_norm": 0.7017843127250671, + "learning_rate": 4.4308667783536946e-05, + "loss": 0.47829113006591795, + "mean_token_accuracy": 0.8566669344902038, + "num_tokens": 2220180.0, + "step": 1400 + }, + { + "entropy": 0.5474000856280327, + "epoch": 3.4365361803084222, + "grad_norm": 0.8140468597412109, + "learning_rate": 4.350981165023561e-05, + "loss": 0.48412700653076174, + "mean_token_accuracy": 0.8574644386768341, + "num_tokens": 2300955.0, + "step": 1450 + }, + { + "entropy": 0.5438253137469292, + "epoch": 3.5551601423487544, + "grad_norm": 0.7523708343505859, + "learning_rate": 4.268135958873494e-05, + "loss": 0.4800748062133789, + "mean_token_accuracy": 0.8587796002626419, + "num_tokens": 2381336.0, + "step": 1500 + }, + { + "entropy": 0.550279560983181, + "epoch": 3.6737841043890866, + "grad_norm": 0.693554162979126, + "learning_rate": 4.182472848634056e-05, + "loss": 0.48448719024658204, + "mean_token_accuracy": 0.8568146306276322, + "num_tokens": 2459186.0, + "step": 1550 + }, + { + "entropy": 0.5500899118185043, + "epoch": 3.792408066429419, + "grad_norm": 0.663986086845398, + "learning_rate": 4.0941383424486856e-05, + "loss": 0.48384044647216795, + "mean_token_accuracy": 0.8548811787366867, + "num_tokens": 2537509.0, + "step": 1600 + }, + { + "entropy": 0.54236083060503, + "epoch": 3.9110320284697506, + "grad_norm": 0.8028485774993896, + "learning_rate": 4.003283517303342e-05, + "loss": 0.4817659378051758, + "mean_token_accuracy": 0.8557876515388488, + "num_tokens": 2617116.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45895950382913064, + "eval_loss": 0.5682657957077026, + "eval_mean_token_accuracy": 0.8392008675469292, + "eval_num_tokens": 2674648.0, + "eval_runtime": 67.4386, + "eval_samples_per_second": 20.211, + "eval_steps_per_second": 2.536, + "step": 1688 + }, + { + "entropy": 0.5392520301269762, + "epoch": 4.02846975088968, + "grad_norm": 0.7537441849708557, + "learning_rate": 3.9100637606421364e-05, + "loss": 0.47076080322265623, + "mean_token_accuracy": 0.8578027938351487, + "num_tokens": 2692377.0, + "step": 1700 + }, + { + "entropy": 0.4877688565850258, + "epoch": 4.147093712930012, + "grad_norm": 0.8088716864585876, + "learning_rate": 3.814638504610852e-05, + "loss": 0.414734992980957, + "mean_token_accuracy": 0.8727810794115066, + "num_tokens": 2772306.0, + "step": 1750 + }, + { + "entropy": 0.47820613652467725, + "epoch": 4.265717674970344, + "grad_norm": 0.7463680505752563, + "learning_rate": 3.7171709533829017e-05, + "loss": 0.41046337127685545, + "mean_token_accuracy": 0.8740614771842956, + "num_tokens": 2851944.0, + "step": 1800 + }, + { + "entropy": 0.48805462688207624, + "epoch": 4.384341637010676, + "grad_norm": 0.7413962483406067, + "learning_rate": 3.617827804034028e-05, + "loss": 0.41806373596191404, + "mean_token_accuracy": 0.8710768383741379, + "num_tokens": 2932030.0, + "step": 1850 + }, + { + "entropy": 0.49493742316961287, + "epoch": 4.5029655990510085, + "grad_norm": 0.648345947265625, + "learning_rate": 3.5167789614431665e-05, + "loss": 0.4231264114379883, + "mean_token_accuracy": 0.8715304219722748, + "num_tokens": 3011165.0, + "step": 1900 + }, + { + "entropy": 0.48624082893133164, + "epoch": 4.621589561091341, + "grad_norm": 0.9829961657524109, + "learning_rate": 3.414197247707055e-05, + "loss": 0.41783756256103516, + "mean_token_accuracy": 0.8720235615968704, + "num_tokens": 3091821.0, + "step": 1950 + }, + { + "entropy": 0.4882233440876007, + "epoch": 4.740213523131673, + "grad_norm": 0.7564496397972107, + "learning_rate": 3.310258106565575e-05, + "loss": 0.4199684524536133, + "mean_token_accuracy": 0.8720525479316712, + "num_tokens": 3169927.0, + "step": 2000 + }, + { + "entropy": 0.4850278961658478, + "epoch": 4.858837485172005, + "grad_norm": 0.9846013188362122, + "learning_rate": 3.2051393033433424e-05, + "loss": 0.41581527709960936, + "mean_token_accuracy": 0.8731251412630081, + "num_tokens": 3248550.0, + "step": 2050 + }, + { + "entropy": 0.49007617592811586, + "epoch": 4.977461447212336, + "grad_norm": 0.9568490386009216, + "learning_rate": 3.099020620920731e-05, + "loss": 0.42266311645507815, + "mean_token_accuracy": 0.8716179597377777, + "num_tokens": 3328838.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4346666379630217, + "eval_loss": 0.5817540287971497, + "eval_mean_token_accuracy": 0.8398688543609708, + "eval_num_tokens": 3343310.0, + "eval_runtime": 67.4228, + "eval_samples_per_second": 20.216, + "eval_steps_per_second": 2.536, + "step": 2110 + }, + { + "entropy": 0.44751178706535183, + "epoch": 5.094899169632265, + "grad_norm": 0.8798995614051819, + "learning_rate": 2.9920835522543054e-05, + "loss": 0.3644564437866211, + "mean_token_accuracy": 0.8864451273523196, + "num_tokens": 3406471.0, + "step": 2150 + }, + { + "entropy": 0.4300405251979828, + "epoch": 5.213523131672598, + "grad_norm": 0.9511876106262207, + "learning_rate": 2.8845109899725434e-05, + "loss": 0.35093097686767577, + "mean_token_accuracy": 0.8902131456136704, + "num_tokens": 3484267.0, + "step": 2200 + }, + { + "entropy": 0.4232933124899864, + "epoch": 5.33214709371293, + "grad_norm": 0.9091081619262695, + "learning_rate": 2.776486913577716e-05, + "loss": 0.35009231567382815, + "mean_token_accuracy": 0.8907366544008255, + "num_tokens": 3562108.0, + "step": 2250 + }, + { + "entropy": 0.42291093826293946, + "epoch": 5.450771055753262, + "grad_norm": 0.9557301998138428, + "learning_rate": 2.6681960747889137e-05, + "loss": 0.3484030532836914, + "mean_token_accuracy": 0.8915014320611954, + "num_tokens": 3643361.0, + "step": 2300 + }, + { + "entropy": 0.42662834584712983, + "epoch": 5.569395017793594, + "grad_norm": 1.0737801790237427, + "learning_rate": 2.559823681564358e-05, + "loss": 0.3529233551025391, + "mean_token_accuracy": 0.890388081073761, + "num_tokens": 3722789.0, + "step": 2350 + }, + { + "entropy": 0.42317517310380937, + "epoch": 5.6880189798339265, + "grad_norm": 0.9151463508605957, + "learning_rate": 2.4515550813434192e-05, + "loss": 0.34859146118164064, + "mean_token_accuracy": 0.8905287456512451, + "num_tokens": 3801491.0, + "step": 2400 + }, + { + "entropy": 0.4059224596619606, + "epoch": 5.806642941874259, + "grad_norm": 1.039889931678772, + "learning_rate": 2.3435754440500813e-05, + "loss": 0.335848388671875, + "mean_token_accuracy": 0.8958643442392349, + "num_tokens": 3883764.0, + "step": 2450 + }, + { + "entropy": 0.4227695450186729, + "epoch": 5.925266903914591, + "grad_norm": 1.0575917959213257, + "learning_rate": 2.2360694454000018e-05, + "loss": 0.34941902160644533, + "mean_token_accuracy": 0.8892073971033096, + "num_tokens": 3963082.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.39420547523693733, + "eval_loss": 0.6033952832221985, + "eval_mean_token_accuracy": 0.8417927235887762, + "eval_num_tokens": 4011972.0, + "eval_runtime": 67.4533, + "eval_samples_per_second": 20.207, + "eval_steps_per_second": 2.535, + "step": 2532 + }, + { + "entropy": 0.40804383550027407, + "epoch": 6.04270462633452, + "grad_norm": 1.0490775108337402, + "learning_rate": 2.129220951052819e-05, + "loss": 0.3283506774902344, + "mean_token_accuracy": 0.8975986519245186, + "num_tokens": 4040122.0, + "step": 2550 + }, + { + "entropy": 0.3564245703816414, + "epoch": 6.161328588374852, + "grad_norm": 1.0544692277908325, + "learning_rate": 2.0232127021498766e-05, + "loss": 0.26940509796142575, + "mean_token_accuracy": 0.915808516740799, + "num_tokens": 4118433.0, + "step": 2600 + }, + { + "entropy": 0.3520303949713707, + "epoch": 6.279952550415183, + "grad_norm": 1.0909168720245361, + "learning_rate": 1.9182260027751904e-05, + "loss": 0.2703816223144531, + "mean_token_accuracy": 0.9140739214420318, + "num_tokens": 4196965.0, + "step": 2650 + }, + { + "entropy": 0.36691532015800477, + "epoch": 6.398576512455516, + "grad_norm": 1.0912492275238037, + "learning_rate": 1.8144404098741954e-05, + "loss": 0.28067663192749026, + "mean_token_accuracy": 0.9112870472669602, + "num_tokens": 4272810.0, + "step": 2700 + }, + { + "entropy": 0.35482348829507826, + "epoch": 6.517200474495848, + "grad_norm": 1.2286453247070312, + "learning_rate": 1.7120334261605962e-05, + "loss": 0.27153289794921875, + "mean_token_accuracy": 0.9141361153125763, + "num_tokens": 4352908.0, + "step": 2750 + }, + { + "entropy": 0.3551847892999649, + "epoch": 6.63582443653618, + "grad_norm": 1.048604965209961, + "learning_rate": 1.611180196536523e-05, + "loss": 0.26977054595947264, + "mean_token_accuracy": 0.9142266172170639, + "num_tokens": 4431727.0, + "step": 2800 + }, + { + "entropy": 0.34558281034231186, + "epoch": 6.754448398576512, + "grad_norm": 0.9390625953674316, + "learning_rate": 1.51205320854522e-05, + "loss": 0.26851280212402345, + "mean_token_accuracy": 0.9141441309452056, + "num_tokens": 4512844.0, + "step": 2850 + }, + { + "entropy": 0.35204614490270614, + "epoch": 6.8730723606168445, + "grad_norm": 0.9959868788719177, + "learning_rate": 1.4148219973685769e-05, + "loss": 0.2707656478881836, + "mean_token_accuracy": 0.9134996753931045, + "num_tokens": 4594732.0, + "step": 2900 + }, + { + "entropy": 0.3469261267781258, + "epoch": 6.991696322657177, + "grad_norm": 0.9221636652946472, + "learning_rate": 1.3196528558740103e-05, + "loss": 0.269027099609375, + "mean_token_accuracy": 0.9142447954416275, + "num_tokens": 4674745.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.33269924674814905, + "eval_loss": 0.7036522626876831, + "eval_mean_token_accuracy": 0.8354723962426883, + "eval_num_tokens": 4680634.0, + "eval_runtime": 67.4408, + "eval_samples_per_second": 20.21, + "eval_steps_per_second": 2.536, + "step": 2954 + }, + { + "entropy": 0.3100580937332577, + "epoch": 7.109134045077106, + "grad_norm": 1.0155000686645508, + "learning_rate": 1.2267085502066457e-05, + "loss": 0.20666887283325194, + "mean_token_accuracy": 0.9355484825192075, + "num_tokens": 4751686.0, + "step": 3000 + }, + { + "entropy": 0.28997830346226694, + "epoch": 7.227758007117438, + "grad_norm": 1.0495970249176025, + "learning_rate": 1.1361480414131762e-05, + "loss": 0.19952856063842772, + "mean_token_accuracy": 0.9384147107601166, + "num_tokens": 4832492.0, + "step": 3050 + }, + { + "entropy": 0.30070391461253165, + "epoch": 7.34638196915777, + "grad_norm": 1.3204537630081177, + "learning_rate": 1.0481262135735177e-05, + "loss": 0.2062089729309082, + "mean_token_accuracy": 0.936167967915535, + "num_tokens": 4910805.0, + "step": 3100 + }, + { + "entropy": 0.289249883890152, + "epoch": 7.465005931198102, + "grad_norm": 1.131684422492981, + "learning_rate": 9.627936089052345e-06, + "loss": 0.19861122131347655, + "mean_token_accuracy": 0.9381200420856476, + "num_tokens": 4992483.0, + "step": 3150 + }, + { + "entropy": 0.2961649990081787, + "epoch": 7.5836298932384345, + "grad_norm": 1.060278296470642, + "learning_rate": 8.802961702937752e-06, + "loss": 0.2034711265563965, + "mean_token_accuracy": 0.9352211368083954, + "num_tokens": 5071712.0, + "step": 3200 + }, + { + "entropy": 0.2959682309627533, + "epoch": 7.702253855278766, + "grad_norm": 1.09508216381073, + "learning_rate": 8.007749916888647e-06, + "loss": 0.20609720230102538, + "mean_token_accuracy": 0.935073122382164, + "num_tokens": 5149157.0, + "step": 3250 + }, + { + "entropy": 0.2906328068673611, + "epoch": 7.820877817319099, + "grad_norm": 0.9429389834403992, + "learning_rate": 7.243660767939519e-06, + "loss": 0.19851903915405272, + "mean_token_accuracy": 0.9371049195528031, + "num_tokens": 5230144.0, + "step": 3300 + }, + { + "entropy": 0.2917777775228024, + "epoch": 7.93950177935943, + "grad_norm": 0.9275627136230469, + "learning_rate": 6.512001064614148e-06, + "loss": 0.20342073440551758, + "mean_token_accuracy": 0.9363774961233139, + "num_tokens": 5309214.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.29100701056028666, + "eval_loss": 0.8126235008239746, + "eval_mean_token_accuracy": 0.8302465771373949, + "eval_num_tokens": 5349296.0, + "eval_runtime": 67.4144, + "eval_samples_per_second": 20.218, + "eval_steps_per_second": 2.537, + "step": 3376 + }, + { + "entropy": 0.27969253860940835, + "epoch": 8.05693950177936, + "grad_norm": 1.0733296871185303, + "learning_rate": 5.814022151913483e-06, + "loss": 0.18230682373046875, + "mean_token_accuracy": 0.9442709967343494, + "num_tokens": 5389623.0, + "step": 3400 + }, + { + "entropy": 0.249246060103178, + "epoch": 8.175563463819692, + "grad_norm": 1.1532745361328125, + "learning_rate": 5.150917771161802e-06, + "loss": 0.15249720573425293, + "mean_token_accuracy": 0.954327083826065, + "num_tokens": 5470361.0, + "step": 3450 + }, + { + "entropy": 0.2619244894385338, + "epoch": 8.294187425860024, + "grad_norm": 0.9613165855407715, + "learning_rate": 4.523822018371486e-06, + "loss": 0.1615569305419922, + "mean_token_accuracy": 0.9513245701789856, + "num_tokens": 5546553.0, + "step": 3500 + }, + { + "entropy": 0.25780971825122834, + "epoch": 8.412811387900355, + "grad_norm": 0.9987137317657471, + "learning_rate": 3.93380740461811e-06, + "loss": 0.16112512588500977, + "mean_token_accuracy": 0.9494028472900391, + "num_tokens": 5623387.0, + "step": 3550 + }, + { + "entropy": 0.25041655868291857, + "epoch": 8.531435349940688, + "grad_norm": 0.9792889356613159, + "learning_rate": 3.381883021743194e-06, + "loss": 0.15699137687683107, + "mean_token_accuracy": 0.9521727949380875, + "num_tokens": 5703112.0, + "step": 3600 + }, + { + "entropy": 0.2552810598909855, + "epoch": 8.65005931198102, + "grad_norm": 1.1667588949203491, + "learning_rate": 2.868992816521833e-06, + "loss": 0.15912227630615233, + "mean_token_accuracy": 0.9508797514438629, + "num_tokens": 5782342.0, + "step": 3650 + }, + { + "entropy": 0.2524907249212265, + "epoch": 8.768683274021353, + "grad_norm": 1.2174683809280396, + "learning_rate": 2.3960139762467253e-06, + "loss": 0.1558846950531006, + "mean_token_accuracy": 0.9524789917469024, + "num_tokens": 5860871.0, + "step": 3700 + }, + { + "entropy": 0.24737386167049408, + "epoch": 8.887307236061684, + "grad_norm": 1.1920585632324219, + "learning_rate": 1.963755428489844e-06, + "loss": 0.15519388198852538, + "mean_token_accuracy": 0.9529236805438995, + "num_tokens": 5941467.0, + "step": 3750 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2624759337706873, + "eval_loss": 0.9007481336593628, + "eval_mean_token_accuracy": 0.8296350861153408, + "eval_num_tokens": 6017958.0, + "eval_runtime": 67.3932, + "eval_samples_per_second": 20.225, + "eval_steps_per_second": 2.537, + "step": 3798 + }, + { + "entropy": 0.25232172208000914, + "epoch": 9.004744958481613, + "grad_norm": 0.7662824392318726, + "learning_rate": 1.5729564576075673e-06, + "loss": 0.15981536865234375, + "mean_token_accuracy": 0.9522925639393354, + "num_tokens": 6021200.0, + "step": 3800 + }, + { + "entropy": 0.23839749366045, + "epoch": 9.123368920521946, + "grad_norm": 0.894930899143219, + "learning_rate": 1.2242854403553214e-06, + "loss": 0.13674766540527344, + "mean_token_accuracy": 0.9596977865695954, + "num_tokens": 6101961.0, + "step": 3850 + }, + { + "entropy": 0.23259827136993408, + "epoch": 9.241992882562277, + "grad_norm": 0.8067717552185059, + "learning_rate": 9.183387027743311e-07, + "loss": 0.1351235866546631, + "mean_token_accuracy": 0.9600261461734771, + "num_tokens": 6181709.0, + "step": 3900 + }, + { + "entropy": 0.2356501851975918, + "epoch": 9.36061684460261, + "grad_norm": 0.896894633769989, + "learning_rate": 6.556395003054744e-07, + "loss": 0.1389451217651367, + "mean_token_accuracy": 0.9588282018899917, + "num_tokens": 6261119.0, + "step": 3950 + }, + { + "entropy": 0.23196498304605484, + "epoch": 9.479240806642942, + "grad_norm": 0.8581342101097107, + "learning_rate": 4.3663712287446603e-07, + "loss": 0.13365538597106932, + "mean_token_accuracy": 0.9593761909008026, + "num_tokens": 6340473.0, + "step": 4000 + }, + { + "entropy": 0.2363830165565014, + "epoch": 9.597864768683275, + "grad_norm": 0.9254534244537354, + "learning_rate": 2.6170612647905447e-07, + "loss": 0.13845305442810057, + "mean_token_accuracy": 0.9584839338064194, + "num_tokens": 6417563.0, + "step": 4050 + }, + { + "entropy": 0.24113454475998877, + "epoch": 9.716488730723606, + "grad_norm": 1.174904465675354, + "learning_rate": 1.3114569259234822e-07, + "loss": 0.14188508987426757, + "mean_token_accuracy": 0.958064352273941, + "num_tokens": 6494348.0, + "step": 4100 + }, + { + "entropy": 0.23012797608971597, + "epoch": 9.83511269276394, + "grad_norm": 0.9481993913650513, + "learning_rate": 4.517911647786601e-08, + "loss": 0.13527493476867675, + "mean_token_accuracy": 0.9589405173063278, + "num_tokens": 6576222.0, + "step": 4150 + }, + { + "entropy": 0.23646119579672814, + "epoch": 9.95373665480427, + "grad_norm": 1.0326144695281982, + "learning_rate": 3.953425291519001e-09, + "loss": 0.137552433013916, + "mean_token_accuracy": 0.959097443819046, + "num_tokens": 6655468.0, + "step": 4200 + }, + { + "epoch": 10.0, + "eval_entropy": 0.251476878049778, + "eval_loss": 0.9530177116394043, + "eval_mean_token_accuracy": 0.8285936964185614, + "eval_num_tokens": 6686620.0, + "eval_runtime": 67.4362, + "eval_samples_per_second": 20.212, + "eval_steps_per_second": 2.536, + "step": 4220 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3.1004012761369805e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/README.md b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/adapter_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9bab0f80ed8a84854ceb2cadaf8ad0a5316e3c --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.09665628722907783, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "up_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/chat_template.jinja b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/tokenizer_config.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/trainer_state.json b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ba23ebed22c243304ad06029766d0059856ff3b3 --- /dev/null +++ b/productivity_code_Estonian/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test2/checkpoint-844/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 844, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 3.1416092801094053, + "epoch": 0.11862396204033215, + "grad_norm": 1.9392393827438354, + "learning_rate": 6.08554758780814e-06, + "loss": 3.097901611328125, + "mean_token_accuracy": 0.4968676868081093, + "num_tokens": 79249.0, + "step": 50 + }, + { + "entropy": 2.040227035284042, + "epoch": 0.2372479240806643, + "grad_norm": 1.3242024183273315, + "learning_rate": 1.2295290024347057e-05, + "loss": 1.9382949829101563, + "mean_token_accuracy": 0.6193598276376724, + "num_tokens": 158897.0, + "step": 100 + }, + { + "entropy": 1.0607982391119004, + "epoch": 0.35587188612099646, + "grad_norm": 1.3427504301071167, + "learning_rate": 1.8505032460885976e-05, + "loss": 1.0002214813232422, + "mean_token_accuracy": 0.7435284560918808, + "num_tokens": 242655.0, + "step": 150 + }, + { + "entropy": 0.9236356592178345, + "epoch": 0.4744958481613286, + "grad_norm": 1.219579815864563, + "learning_rate": 2.4714774897424894e-05, + "loss": 0.8791838836669922, + "mean_token_accuracy": 0.7669315725564957, + "num_tokens": 318816.0, + "step": 200 + }, + { + "entropy": 0.8616744816303253, + "epoch": 0.5931198102016607, + "grad_norm": 1.138848900794983, + "learning_rate": 3.092451733396381e-05, + "loss": 0.8118074798583984, + "mean_token_accuracy": 0.7797698676586151, + "num_tokens": 397671.0, + "step": 250 + }, + { + "entropy": 0.8212076133489609, + "epoch": 0.7117437722419929, + "grad_norm": 1.166795253753662, + "learning_rate": 3.713425977050273e-05, + "loss": 0.764132080078125, + "mean_token_accuracy": 0.7912415993213654, + "num_tokens": 473156.0, + "step": 300 + }, + { + "entropy": 0.7789461934566497, + "epoch": 0.830367734282325, + "grad_norm": 1.0946475267410278, + "learning_rate": 4.334400220704165e-05, + "loss": 0.7244119262695312, + "mean_token_accuracy": 0.8039911246299744, + "num_tokens": 554823.0, + "step": 350 + }, + { + "entropy": 0.7613714265823365, + "epoch": 0.9489916963226572, + "grad_norm": 0.755024790763855, + "learning_rate": 4.9553744643580566e-05, + "loss": 0.7079727172851562, + "mean_token_accuracy": 0.8072243589162826, + "num_tokens": 634813.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.63530013819187, + "eval_loss": 0.6358877420425415, + "eval_mean_token_accuracy": 0.8174427409618221, + "eval_num_tokens": 668662.0, + "eval_runtime": 67.9589, + "eval_samples_per_second": 20.056, + "eval_steps_per_second": 2.516, + "step": 422 + }, + { + "entropy": 0.7371636526753204, + "epoch": 1.066429418742586, + "grad_norm": 0.7297853231430054, + "learning_rate": 5.240369101765075e-05, + "loss": 0.6795420837402344, + "mean_token_accuracy": 0.8091908283907958, + "num_tokens": 711999.0, + "step": 450 + }, + { + "entropy": 0.7101516282558441, + "epoch": 1.1850533807829182, + "grad_norm": 0.7399303317070007, + "learning_rate": 5.2357091187486714e-05, + "loss": 0.6535377502441406, + "mean_token_accuracy": 0.8180439311265946, + "num_tokens": 792949.0, + "step": 500 + }, + { + "entropy": 0.7053145176172256, + "epoch": 1.3036773428232502, + "grad_norm": 0.9309378266334534, + "learning_rate": 5.226576407916573e-05, + "loss": 0.6420201110839844, + "mean_token_accuracy": 0.8200493824481964, + "num_tokens": 872947.0, + "step": 550 + }, + { + "entropy": 0.7080373674631119, + "epoch": 1.4223013048635824, + "grad_norm": 0.6808303594589233, + "learning_rate": 5.212986588786971e-05, + "loss": 0.6391039276123047, + "mean_token_accuracy": 0.8223454594612122, + "num_tokens": 948140.0, + "step": 600 + }, + { + "entropy": 0.6866704186797142, + "epoch": 1.5409252669039146, + "grad_norm": 0.654197096824646, + "learning_rate": 5.194962903793681e-05, + "loss": 0.6234166717529297, + "mean_token_accuracy": 0.8264459353685379, + "num_tokens": 1028421.0, + "step": 650 + }, + { + "entropy": 0.6787189674377442, + "epoch": 1.6595492289442468, + "grad_norm": 0.662652850151062, + "learning_rate": 5.172536178535015e-05, + "loss": 0.6130905151367188, + "mean_token_accuracy": 0.8273760837316513, + "num_tokens": 1108931.0, + "step": 700 + }, + { + "entropy": 0.6839700847864151, + "epoch": 1.778173190984579, + "grad_norm": 0.6859784126281738, + "learning_rate": 5.145744769053287e-05, + "loss": 0.6151719284057617, + "mean_token_accuracy": 0.8260005974769592, + "num_tokens": 1187404.0, + "step": 750 + }, + { + "entropy": 0.6775795823335647, + "epoch": 1.896797153024911, + "grad_norm": 0.580604612827301, + "learning_rate": 5.114634496235134e-05, + "loss": 0.6077556228637695, + "mean_token_accuracy": 0.8290794855356216, + "num_tokens": 1267193.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.555681184892766, + "eval_loss": 0.5672050714492798, + "eval_mean_token_accuracy": 0.8383949548877471, + "eval_num_tokens": 1337324.0, + "eval_runtime": 67.3942, + "eval_samples_per_second": 20.224, + "eval_steps_per_second": 2.537, + "step": 844 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.194798765237453e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/README.md b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/adapter_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..22fe9e5a1d9f895d47239aa10bc78cc896f11beb --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.025388885152548692, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "k_proj", + "gate_proj", + "o_proj", + "q_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/chat_template.jinja b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/tokenizer_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/trainer_state.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..df86a39ae97acb99a71ee5b0c147d02828ba66ed --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-2912/trainer_state.json @@ -0,0 +1,691 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2912, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.8487297534942626, + "epoch": 0.12033694344163658, + "grad_norm": 1.1108952760696411, + "learning_rate": 3.403535898928475e-05, + "loss": 2.800888977050781, + "mean_token_accuracy": 0.5476393702626229, + "num_tokens": 80236.0, + "step": 50 + }, + { + "entropy": 1.3183529460430146, + "epoch": 0.24067388688327315, + "grad_norm": 0.9604457020759583, + "learning_rate": 6.876531714161613e-05, + "loss": 1.2430918884277344, + "mean_token_accuracy": 0.7165319395065307, + "num_tokens": 158954.0, + "step": 100 + }, + { + "entropy": 0.7940413594245911, + "epoch": 0.36101083032490977, + "grad_norm": 0.6496508717536926, + "learning_rate": 0.00010349527529394752, + "loss": 0.7494664001464844, + "mean_token_accuracy": 0.7958653098344803, + "num_tokens": 235314.0, + "step": 150 + }, + { + "entropy": 0.7416071015596389, + "epoch": 0.4813477737665463, + "grad_norm": 0.7964484095573425, + "learning_rate": 0.0001382252334462789, + "loss": 0.6981658935546875, + "mean_token_accuracy": 0.8044728142023087, + "num_tokens": 311529.0, + "step": 200 + }, + { + "entropy": 0.6903449124097825, + "epoch": 0.601684717208183, + "grad_norm": 0.6642301082611084, + "learning_rate": 0.00017295519159861027, + "loss": 0.6478567504882813, + "mean_token_accuracy": 0.8178131663799286, + "num_tokens": 386903.0, + "step": 250 + }, + { + "entropy": 0.6663334459066391, + "epoch": 0.7220216606498195, + "grad_norm": 0.5385578274726868, + "learning_rate": 0.00020768514975094167, + "loss": 0.617872200012207, + "mean_token_accuracy": 0.827698912024498, + "num_tokens": 461514.0, + "step": 300 + }, + { + "entropy": 0.6437311708927155, + "epoch": 0.8423586040914561, + "grad_norm": 0.4748707413673401, + "learning_rate": 0.00024241510790327304, + "loss": 0.5984851455688477, + "mean_token_accuracy": 0.8346574056148529, + "num_tokens": 536592.0, + "step": 350 + }, + { + "entropy": 0.6274739676713943, + "epoch": 0.9626955475330926, + "grad_norm": 0.5358314514160156, + "learning_rate": 0.0002771450660556044, + "loss": 0.5809464263916015, + "mean_token_accuracy": 0.8380268800258637, + "num_tokens": 609718.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.55972887462453, + "eval_loss": 0.5503765940666199, + "eval_mean_token_accuracy": 0.8400089623807352, + "eval_num_tokens": 632286.0, + "eval_runtime": 57.4057, + "eval_samples_per_second": 22.001, + "eval_steps_per_second": 2.752, + "step": 416 + }, + { + "entropy": 0.6137794244168985, + "epoch": 1.0818291215403129, + "grad_norm": 0.48725026845932007, + "learning_rate": 0.00028889786636446425, + "loss": 0.5690413665771484, + "mean_token_accuracy": 0.8418765904927494, + "num_tokens": 683904.0, + "step": 450 + }, + { + "entropy": 0.5970469158887863, + "epoch": 1.2021660649819494, + "grad_norm": 0.5252997279167175, + "learning_rate": 0.0002886030033198732, + "loss": 0.5451018524169922, + "mean_token_accuracy": 0.8463934576511383, + "num_tokens": 758298.0, + "step": 500 + }, + { + "entropy": 0.5896219766139984, + "epoch": 1.322503008423586, + "grad_norm": 0.3888632655143738, + "learning_rate": 0.00028805448269855045, + "loss": 0.5395643997192383, + "mean_token_accuracy": 0.8473994207382202, + "num_tokens": 836432.0, + "step": 550 + }, + { + "entropy": 0.5761718067526818, + "epoch": 1.4428399518652226, + "grad_norm": 0.40175220370292664, + "learning_rate": 0.0002872532698783407, + "loss": 0.531209602355957, + "mean_token_accuracy": 0.8493033695220947, + "num_tokens": 912728.0, + "step": 600 + }, + { + "entropy": 0.5754709538817405, + "epoch": 1.5631768953068592, + "grad_norm": 0.34686869382858276, + "learning_rate": 0.0002862007749668598, + "loss": 0.529142074584961, + "mean_token_accuracy": 0.8509961807727814, + "num_tokens": 989711.0, + "step": 650 + }, + { + "entropy": 0.564461555480957, + "epoch": 1.6835138387484958, + "grad_norm": 0.3425906002521515, + "learning_rate": 0.0002848988503197527, + "loss": 0.5160605239868165, + "mean_token_accuracy": 0.8546184468269348, + "num_tokens": 1065292.0, + "step": 700 + }, + { + "entropy": 0.5677110701799393, + "epoch": 1.8038507821901324, + "grad_norm": 0.34574612975120544, + "learning_rate": 0.00028334978728060975, + "loss": 0.5171365737915039, + "mean_token_accuracy": 0.8528753453493119, + "num_tokens": 1141585.0, + "step": 750 + }, + { + "entropy": 0.5702121770381927, + "epoch": 1.924187725631769, + "grad_norm": 0.39626583456993103, + "learning_rate": 0.0002815563121482791, + "loss": 0.5194630432128906, + "mean_token_accuracy": 0.8525324720144272, + "num_tokens": 1217261.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5043281727953802, + "eval_loss": 0.5085281133651733, + "eval_mean_token_accuracy": 0.8485767777207531, + "eval_num_tokens": 1264572.0, + "eval_runtime": 57.6896, + "eval_samples_per_second": 21.893, + "eval_steps_per_second": 2.739, + "step": 832 + }, + { + "entropy": 0.5520541523442124, + "epoch": 2.0433212996389893, + "grad_norm": 0.34861499071121216, + "learning_rate": 0.00027952158137867244, + "loss": 0.4953007125854492, + "mean_token_accuracy": 0.8596346799773399, + "num_tokens": 1292148.0, + "step": 850 + }, + { + "entropy": 0.5117866912484169, + "epoch": 2.1636582430806257, + "grad_norm": 0.4384036958217621, + "learning_rate": 0.0002772491760295088, + "loss": 0.4596589660644531, + "mean_token_accuracy": 0.8644879817962646, + "num_tokens": 1368098.0, + "step": 900 + }, + { + "entropy": 0.5110387325286865, + "epoch": 2.2839951865222625, + "grad_norm": 0.44647786021232605, + "learning_rate": 0.0002747430954577733, + "loss": 0.46299396514892577, + "mean_token_accuracy": 0.8637035477161408, + "num_tokens": 1444959.0, + "step": 950 + }, + { + "entropy": 0.5270229122042656, + "epoch": 2.404332129963899, + "grad_norm": 0.3942628502845764, + "learning_rate": 0.00027200775028098345, + "loss": 0.4726287078857422, + "mean_token_accuracy": 0.8627178031206131, + "num_tokens": 1516670.0, + "step": 1000 + }, + { + "entropy": 0.518384268283844, + "epoch": 2.5246690734055353, + "grad_norm": 0.4096919298171997, + "learning_rate": 0.0002690479546146505, + "loss": 0.463576545715332, + "mean_token_accuracy": 0.863879257440567, + "num_tokens": 1592121.0, + "step": 1050 + }, + { + "entropy": 0.5208215075731277, + "epoch": 2.645006016847172, + "grad_norm": 0.32399967312812805, + "learning_rate": 0.0002658689175995985, + "loss": 0.47119388580322263, + "mean_token_accuracy": 0.8642315500974656, + "num_tokens": 1670766.0, + "step": 1100 + }, + { + "entropy": 0.5203104516863823, + "epoch": 2.765342960288809, + "grad_norm": 0.32846879959106445, + "learning_rate": 0.0002624762342340511, + "loss": 0.47053630828857423, + "mean_token_accuracy": 0.8640051782131195, + "num_tokens": 1746321.0, + "step": 1150 + }, + { + "entropy": 0.5191598236560822, + "epoch": 2.8856799037304453, + "grad_norm": 0.36446335911750793, + "learning_rate": 0.0002588758755266231, + "loss": 0.46290687561035154, + "mean_token_accuracy": 0.8656695383787155, + "num_tokens": 1823845.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.48268982862369925, + "eval_loss": 0.4935414791107178, + "eval_mean_token_accuracy": 0.8550615201268015, + "eval_num_tokens": 1896858.0, + "eval_runtime": 57.3895, + "eval_samples_per_second": 22.008, + "eval_steps_per_second": 2.753, + "step": 1248 + }, + { + "entropy": 0.5235216328592012, + "epoch": 3.0048134777376654, + "grad_norm": 0.3717617690563202, + "learning_rate": 0.00025507417798754624, + "loss": 0.46804916381835937, + "mean_token_accuracy": 0.8647771068293639, + "num_tokens": 1899333.0, + "step": 1250 + }, + { + "entropy": 0.4668200609087944, + "epoch": 3.125150421179302, + "grad_norm": 0.37964364886283875, + "learning_rate": 0.0002510778324766242, + "loss": 0.4062967300415039, + "mean_token_accuracy": 0.8777556318044663, + "num_tokens": 1974386.0, + "step": 1300 + }, + { + "entropy": 0.4731095978617668, + "epoch": 3.2454873646209386, + "grad_norm": 0.4152783453464508, + "learning_rate": 0.0002468938724275443, + "loss": 0.4166794586181641, + "mean_token_accuracy": 0.875180395245552, + "num_tokens": 2046775.0, + "step": 1350 + }, + { + "entropy": 0.47036315232515336, + "epoch": 3.3658243080625754, + "grad_norm": 0.41300854086875916, + "learning_rate": 0.0002425296614692708, + "loss": 0.41332015991210935, + "mean_token_accuracy": 0.8770327359437943, + "num_tokens": 2120417.0, + "step": 1400 + }, + { + "entropy": 0.46202471256256106, + "epoch": 3.4861612515042117, + "grad_norm": 0.353500634431839, + "learning_rate": 0.0002379928804663058, + "loss": 0.4094076156616211, + "mean_token_accuracy": 0.8768983954191207, + "num_tokens": 2198640.0, + "step": 1450 + }, + { + "entropy": 0.45835661113262177, + "epoch": 3.606498194945848, + "grad_norm": 0.45422980189323425, + "learning_rate": 0.0002332915140006257, + "loss": 0.40608222961425783, + "mean_token_accuracy": 0.8777871477603912, + "num_tokens": 2277538.0, + "step": 1500 + }, + { + "entropy": 0.4667848441004753, + "epoch": 3.726835138387485, + "grad_norm": 0.3852768838405609, + "learning_rate": 0.00022843383631908587, + "loss": 0.41128265380859375, + "mean_token_accuracy": 0.8771094262599946, + "num_tokens": 2355173.0, + "step": 1550 + }, + { + "entropy": 0.462973136305809, + "epoch": 3.8471720818291217, + "grad_norm": 0.5206069946289062, + "learning_rate": 0.00022342839677102398, + "loss": 0.40684612274169923, + "mean_token_accuracy": 0.8784542739391327, + "num_tokens": 2433855.0, + "step": 1600 + }, + { + "entropy": 0.47388757914304735, + "epoch": 3.967509025270758, + "grad_norm": 0.45729249715805054, + "learning_rate": 0.00021828400476169333, + "loss": 0.4176290512084961, + "mean_token_accuracy": 0.8743184685707093, + "num_tokens": 2509050.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.42192115738422054, + "eval_loss": 0.5213983058929443, + "eval_mean_token_accuracy": 0.8507144964193996, + "eval_num_tokens": 2529144.0, + "eval_runtime": 57.4742, + "eval_samples_per_second": 21.975, + "eval_steps_per_second": 2.749, + "step": 1664 + }, + { + "entropy": 0.4243522891492555, + "epoch": 4.086642599277979, + "grad_norm": 0.4242376387119293, + "learning_rate": 0.00021300971424800534, + "loss": 0.3608149719238281, + "mean_token_accuracy": 0.8896295458379418, + "num_tokens": 2584990.0, + "step": 1700 + }, + { + "entropy": 0.39899118691682817, + "epoch": 4.206979542719615, + "grad_norm": 0.527614176273346, + "learning_rate": 0.00020761480780387022, + "loss": 0.3385160446166992, + "mean_token_accuracy": 0.8941174983978272, + "num_tokens": 2660701.0, + "step": 1750 + }, + { + "entropy": 0.40566804468631745, + "epoch": 4.327316486161251, + "grad_norm": 0.4498025178909302, + "learning_rate": 0.00020210878028317902, + "loss": 0.34584579467773435, + "mean_token_accuracy": 0.8929324275255204, + "num_tokens": 2736960.0, + "step": 1800 + }, + { + "entropy": 0.4153370544314384, + "epoch": 4.447653429602888, + "grad_norm": 0.42169710993766785, + "learning_rate": 0.00019650132210918006, + "loss": 0.35420520782470705, + "mean_token_accuracy": 0.8916839396953583, + "num_tokens": 2812402.0, + "step": 1850 + }, + { + "entropy": 0.41663775861263275, + "epoch": 4.567990373044525, + "grad_norm": 0.43042847514152527, + "learning_rate": 0.00019080230221965998, + "loss": 0.3560990524291992, + "mean_token_accuracy": 0.8905349797010422, + "num_tokens": 2888713.0, + "step": 1900 + }, + { + "entropy": 0.4153156217932701, + "epoch": 4.688327316486161, + "grad_norm": 0.6334472298622131, + "learning_rate": 0.00018502175069794524, + "loss": 0.35555774688720704, + "mean_token_accuracy": 0.8908489847183227, + "num_tokens": 2966846.0, + "step": 1950 + }, + { + "entropy": 0.4233098104596138, + "epoch": 4.808664259927798, + "grad_norm": 0.4085949957370758, + "learning_rate": 0.00017916984112029274, + "loss": 0.36258663177490236, + "mean_token_accuracy": 0.8889626759290695, + "num_tokens": 3041477.0, + "step": 2000 + }, + { + "entropy": 0.41765743762254715, + "epoch": 4.929001203369435, + "grad_norm": 0.4329089820384979, + "learning_rate": 0.00017325687265073777, + "loss": 0.35529815673828125, + "mean_token_accuracy": 0.8905666440725326, + "num_tokens": 3114886.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.38224964202204836, + "eval_loss": 0.5339241027832031, + "eval_mean_token_accuracy": 0.8526725293714789, + "eval_num_tokens": 3161430.0, + "eval_runtime": 57.4581, + "eval_samples_per_second": 21.981, + "eval_steps_per_second": 2.75, + "step": 2080 + }, + { + "entropy": 0.38331641332067623, + "epoch": 5.048134777376655, + "grad_norm": 0.5011507868766785, + "learning_rate": 0.00016729325191491184, + "loss": 0.320716552734375, + "mean_token_accuracy": 0.9005015263653765, + "num_tokens": 3191728.0, + "step": 2100 + }, + { + "entropy": 0.34260627657175063, + "epoch": 5.1684717208182915, + "grad_norm": 0.5449230074882507, + "learning_rate": 0.00016128947468473123, + "loss": 0.27347917556762696, + "mean_token_accuracy": 0.9137293487787247, + "num_tokens": 3268401.0, + "step": 2150 + }, + { + "entropy": 0.33992124766111376, + "epoch": 5.2888086642599275, + "grad_norm": 0.5042031407356262, + "learning_rate": 0.0001552561074061919, + "loss": 0.2745149040222168, + "mean_token_accuracy": 0.9130180728435516, + "num_tokens": 3345743.0, + "step": 2200 + }, + { + "entropy": 0.3468990367650986, + "epoch": 5.409145607701564, + "grad_norm": 0.4478400647640228, + "learning_rate": 0.00014920376860277964, + "loss": 0.2810557746887207, + "mean_token_accuracy": 0.910719353556633, + "num_tokens": 3419875.0, + "step": 2250 + }, + { + "entropy": 0.35029833555221557, + "epoch": 5.529482551143201, + "grad_norm": 0.5645950436592102, + "learning_rate": 0.0001431431101872259, + "loss": 0.28505165100097657, + "mean_token_accuracy": 0.91094852745533, + "num_tokens": 3494189.0, + "step": 2300 + }, + { + "entropy": 0.35068276852369307, + "epoch": 5.649819494584838, + "grad_norm": 0.4856931269168854, + "learning_rate": 0.00013708479871449959, + "loss": 0.28504167556762694, + "mean_token_accuracy": 0.9086999785900116, + "num_tokens": 3571387.0, + "step": 2350 + }, + { + "entropy": 0.3502148833870888, + "epoch": 5.770156438026474, + "grad_norm": 0.4992657005786896, + "learning_rate": 0.00013103949660902948, + "loss": 0.2827724266052246, + "mean_token_accuracy": 0.9090785968303681, + "num_tokens": 3647716.0, + "step": 2400 + }, + { + "entropy": 0.34631509900093077, + "epoch": 5.890493381468111, + "grad_norm": 0.5657410621643066, + "learning_rate": 0.00012501784339919524, + "loss": 0.2854422378540039, + "mean_token_accuracy": 0.9095450508594513, + "num_tokens": 3725666.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3132856645727459, + "eval_loss": 0.6306682229042053, + "eval_mean_token_accuracy": 0.8453912821751607, + "eval_num_tokens": 3793716.0, + "eval_runtime": 57.4548, + "eval_samples_per_second": 21.983, + "eval_steps_per_second": 2.75, + "step": 2496 + }, + { + "entropy": 0.347719269569474, + "epoch": 6.009626955475331, + "grad_norm": 0.43329548835754395, + "learning_rate": 0.0001190304369921156, + "loss": 0.2800110626220703, + "mean_token_accuracy": 0.9126462815987944, + "num_tokens": 3799943.0, + "step": 2500 + }, + { + "entropy": 0.27945472195744514, + "epoch": 6.129963898916968, + "grad_norm": 0.634467601776123, + "learning_rate": 0.00011308781502168858, + "loss": 0.20203407287597655, + "mean_token_accuracy": 0.9359134542942047, + "num_tokens": 3876465.0, + "step": 2550 + }, + { + "entropy": 0.2845077550411224, + "epoch": 6.250300842358604, + "grad_norm": 0.7450694441795349, + "learning_rate": 0.00010720043630270964, + "loss": 0.2109172248840332, + "mean_token_accuracy": 0.9338040286302567, + "num_tokens": 3951466.0, + "step": 2600 + }, + { + "entropy": 0.2838969850540161, + "epoch": 6.37063778580024, + "grad_norm": 0.7350204586982727, + "learning_rate": 0.00010137866242370994, + "loss": 0.2094452476501465, + "mean_token_accuracy": 0.9322927874326706, + "num_tokens": 4026603.0, + "step": 2650 + }, + { + "entropy": 0.28683102265000343, + "epoch": 6.490974729241877, + "grad_norm": 0.6212452054023743, + "learning_rate": 9.563273951090865e-05, + "loss": 0.21383323669433593, + "mean_token_accuracy": 0.932304550409317, + "num_tokens": 4102554.0, + "step": 2700 + }, + { + "entropy": 0.27902470380067823, + "epoch": 6.611311672683514, + "grad_norm": 0.6719015836715698, + "learning_rate": 8.997278019537578e-05, + "loss": 0.2090513801574707, + "mean_token_accuracy": 0.9336883723735809, + "num_tokens": 4177827.0, + "step": 2750 + }, + { + "entropy": 0.278264344483614, + "epoch": 6.731648616125151, + "grad_norm": 0.6829904913902283, + "learning_rate": 8.440874581514121e-05, + "loss": 0.20877845764160155, + "mean_token_accuracy": 0.9345877057313919, + "num_tokens": 4253960.0, + "step": 2800 + }, + { + "entropy": 0.2799538014829159, + "epoch": 6.851985559566787, + "grad_norm": 0.71329265832901, + "learning_rate": 7.895042888357504e-05, + "loss": 0.20889324188232422, + "mean_token_accuracy": 0.933683454990387, + "num_tokens": 4332678.0, + "step": 2850 + }, + { + "entropy": 0.2825497867166996, + "epoch": 6.9723225030084235, + "grad_norm": 0.4812498688697815, + "learning_rate": 7.36074358548936e-05, + "loss": 0.21008211135864258, + "mean_token_accuracy": 0.9326896673440933, + "num_tokens": 4409204.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.26715399777587456, + "eval_loss": 0.7001190781593323, + "eval_mean_token_accuracy": 0.8474974322922623, + "eval_num_tokens": 4426002.0, + "eval_runtime": 57.4705, + "eval_samples_per_second": 21.976, + "eval_steps_per_second": 2.749, + "step": 2912 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8489749130594816e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/README.md b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/adapter_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..22fe9e5a1d9f895d47239aa10bc78cc896f11beb --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.025388885152548692, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "k_proj", + "gate_proj", + "o_proj", + "q_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/chat_template.jinja b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/tokenizer_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/trainer_state.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..71fb26ee5670b6a91cff33c2cdd2c66a714aa815 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3328/trainer_state.json @@ -0,0 +1,782 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3328, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.8487297534942626, + "epoch": 0.12033694344163658, + "grad_norm": 1.1108952760696411, + "learning_rate": 3.403535898928475e-05, + "loss": 2.800888977050781, + "mean_token_accuracy": 0.5476393702626229, + "num_tokens": 80236.0, + "step": 50 + }, + { + "entropy": 1.3183529460430146, + "epoch": 0.24067388688327315, + "grad_norm": 0.9604457020759583, + "learning_rate": 6.876531714161613e-05, + "loss": 1.2430918884277344, + "mean_token_accuracy": 0.7165319395065307, + "num_tokens": 158954.0, + "step": 100 + }, + { + "entropy": 0.7940413594245911, + "epoch": 0.36101083032490977, + "grad_norm": 0.6496508717536926, + "learning_rate": 0.00010349527529394752, + "loss": 0.7494664001464844, + "mean_token_accuracy": 0.7958653098344803, + "num_tokens": 235314.0, + "step": 150 + }, + { + "entropy": 0.7416071015596389, + "epoch": 0.4813477737665463, + "grad_norm": 0.7964484095573425, + "learning_rate": 0.0001382252334462789, + "loss": 0.6981658935546875, + "mean_token_accuracy": 0.8044728142023087, + "num_tokens": 311529.0, + "step": 200 + }, + { + "entropy": 0.6903449124097825, + "epoch": 0.601684717208183, + "grad_norm": 0.6642301082611084, + "learning_rate": 0.00017295519159861027, + "loss": 0.6478567504882813, + "mean_token_accuracy": 0.8178131663799286, + "num_tokens": 386903.0, + "step": 250 + }, + { + "entropy": 0.6663334459066391, + "epoch": 0.7220216606498195, + "grad_norm": 0.5385578274726868, + "learning_rate": 0.00020768514975094167, + "loss": 0.617872200012207, + "mean_token_accuracy": 0.827698912024498, + "num_tokens": 461514.0, + "step": 300 + }, + { + "entropy": 0.6437311708927155, + "epoch": 0.8423586040914561, + "grad_norm": 0.4748707413673401, + "learning_rate": 0.00024241510790327304, + "loss": 0.5984851455688477, + "mean_token_accuracy": 0.8346574056148529, + "num_tokens": 536592.0, + "step": 350 + }, + { + "entropy": 0.6274739676713943, + "epoch": 0.9626955475330926, + "grad_norm": 0.5358314514160156, + "learning_rate": 0.0002771450660556044, + "loss": 0.5809464263916015, + "mean_token_accuracy": 0.8380268800258637, + "num_tokens": 609718.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.55972887462453, + "eval_loss": 0.5503765940666199, + "eval_mean_token_accuracy": 0.8400089623807352, + "eval_num_tokens": 632286.0, + "eval_runtime": 57.4057, + "eval_samples_per_second": 22.001, + "eval_steps_per_second": 2.752, + "step": 416 + }, + { + "entropy": 0.6137794244168985, + "epoch": 1.0818291215403129, + "grad_norm": 0.48725026845932007, + "learning_rate": 0.00028889786636446425, + "loss": 0.5690413665771484, + "mean_token_accuracy": 0.8418765904927494, + "num_tokens": 683904.0, + "step": 450 + }, + { + "entropy": 0.5970469158887863, + "epoch": 1.2021660649819494, + "grad_norm": 0.5252997279167175, + "learning_rate": 0.0002886030033198732, + "loss": 0.5451018524169922, + "mean_token_accuracy": 0.8463934576511383, + "num_tokens": 758298.0, + "step": 500 + }, + { + "entropy": 0.5896219766139984, + "epoch": 1.322503008423586, + "grad_norm": 0.3888632655143738, + "learning_rate": 0.00028805448269855045, + "loss": 0.5395643997192383, + "mean_token_accuracy": 0.8473994207382202, + "num_tokens": 836432.0, + "step": 550 + }, + { + "entropy": 0.5761718067526818, + "epoch": 1.4428399518652226, + "grad_norm": 0.40175220370292664, + "learning_rate": 0.0002872532698783407, + "loss": 0.531209602355957, + "mean_token_accuracy": 0.8493033695220947, + "num_tokens": 912728.0, + "step": 600 + }, + { + "entropy": 0.5754709538817405, + "epoch": 1.5631768953068592, + "grad_norm": 0.34686869382858276, + "learning_rate": 0.0002862007749668598, + "loss": 0.529142074584961, + "mean_token_accuracy": 0.8509961807727814, + "num_tokens": 989711.0, + "step": 650 + }, + { + "entropy": 0.564461555480957, + "epoch": 1.6835138387484958, + "grad_norm": 0.3425906002521515, + "learning_rate": 0.0002848988503197527, + "loss": 0.5160605239868165, + "mean_token_accuracy": 0.8546184468269348, + "num_tokens": 1065292.0, + "step": 700 + }, + { + "entropy": 0.5677110701799393, + "epoch": 1.8038507821901324, + "grad_norm": 0.34574612975120544, + "learning_rate": 0.00028334978728060975, + "loss": 0.5171365737915039, + "mean_token_accuracy": 0.8528753453493119, + "num_tokens": 1141585.0, + "step": 750 + }, + { + "entropy": 0.5702121770381927, + "epoch": 1.924187725631769, + "grad_norm": 0.39626583456993103, + "learning_rate": 0.0002815563121482791, + "loss": 0.5194630432128906, + "mean_token_accuracy": 0.8525324720144272, + "num_tokens": 1217261.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5043281727953802, + "eval_loss": 0.5085281133651733, + "eval_mean_token_accuracy": 0.8485767777207531, + "eval_num_tokens": 1264572.0, + "eval_runtime": 57.6896, + "eval_samples_per_second": 21.893, + "eval_steps_per_second": 2.739, + "step": 832 + }, + { + "entropy": 0.5520541523442124, + "epoch": 2.0433212996389893, + "grad_norm": 0.34861499071121216, + "learning_rate": 0.00027952158137867244, + "loss": 0.4953007125854492, + "mean_token_accuracy": 0.8596346799773399, + "num_tokens": 1292148.0, + "step": 850 + }, + { + "entropy": 0.5117866912484169, + "epoch": 2.1636582430806257, + "grad_norm": 0.4384036958217621, + "learning_rate": 0.0002772491760295088, + "loss": 0.4596589660644531, + "mean_token_accuracy": 0.8644879817962646, + "num_tokens": 1368098.0, + "step": 900 + }, + { + "entropy": 0.5110387325286865, + "epoch": 2.2839951865222625, + "grad_norm": 0.44647786021232605, + "learning_rate": 0.0002747430954577733, + "loss": 0.46299396514892577, + "mean_token_accuracy": 0.8637035477161408, + "num_tokens": 1444959.0, + "step": 950 + }, + { + "entropy": 0.5270229122042656, + "epoch": 2.404332129963899, + "grad_norm": 0.3942628502845764, + "learning_rate": 0.00027200775028098345, + "loss": 0.4726287078857422, + "mean_token_accuracy": 0.8627178031206131, + "num_tokens": 1516670.0, + "step": 1000 + }, + { + "entropy": 0.518384268283844, + "epoch": 2.5246690734055353, + "grad_norm": 0.4096919298171997, + "learning_rate": 0.0002690479546146505, + "loss": 0.463576545715332, + "mean_token_accuracy": 0.863879257440567, + "num_tokens": 1592121.0, + "step": 1050 + }, + { + "entropy": 0.5208215075731277, + "epoch": 2.645006016847172, + "grad_norm": 0.32399967312812805, + "learning_rate": 0.0002658689175995985, + "loss": 0.47119388580322263, + "mean_token_accuracy": 0.8642315500974656, + "num_tokens": 1670766.0, + "step": 1100 + }, + { + "entropy": 0.5203104516863823, + "epoch": 2.765342960288809, + "grad_norm": 0.32846879959106445, + "learning_rate": 0.0002624762342340511, + "loss": 0.47053630828857423, + "mean_token_accuracy": 0.8640051782131195, + "num_tokens": 1746321.0, + "step": 1150 + }, + { + "entropy": 0.5191598236560822, + "epoch": 2.8856799037304453, + "grad_norm": 0.36446335911750793, + "learning_rate": 0.0002588758755266231, + "loss": 0.46290687561035154, + "mean_token_accuracy": 0.8656695383787155, + "num_tokens": 1823845.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.48268982862369925, + "eval_loss": 0.4935414791107178, + "eval_mean_token_accuracy": 0.8550615201268015, + "eval_num_tokens": 1896858.0, + "eval_runtime": 57.3895, + "eval_samples_per_second": 22.008, + "eval_steps_per_second": 2.753, + "step": 1248 + }, + { + "entropy": 0.5235216328592012, + "epoch": 3.0048134777376654, + "grad_norm": 0.3717617690563202, + "learning_rate": 0.00025507417798754624, + "loss": 0.46804916381835937, + "mean_token_accuracy": 0.8647771068293639, + "num_tokens": 1899333.0, + "step": 1250 + }, + { + "entropy": 0.4668200609087944, + "epoch": 3.125150421179302, + "grad_norm": 0.37964364886283875, + "learning_rate": 0.0002510778324766242, + "loss": 0.4062967300415039, + "mean_token_accuracy": 0.8777556318044663, + "num_tokens": 1974386.0, + "step": 1300 + }, + { + "entropy": 0.4731095978617668, + "epoch": 3.2454873646209386, + "grad_norm": 0.4152783453464508, + "learning_rate": 0.0002468938724275443, + "loss": 0.4166794586181641, + "mean_token_accuracy": 0.875180395245552, + "num_tokens": 2046775.0, + "step": 1350 + }, + { + "entropy": 0.47036315232515336, + "epoch": 3.3658243080625754, + "grad_norm": 0.41300854086875916, + "learning_rate": 0.0002425296614692708, + "loss": 0.41332015991210935, + "mean_token_accuracy": 0.8770327359437943, + "num_tokens": 2120417.0, + "step": 1400 + }, + { + "entropy": 0.46202471256256106, + "epoch": 3.4861612515042117, + "grad_norm": 0.353500634431839, + "learning_rate": 0.0002379928804663058, + "loss": 0.4094076156616211, + "mean_token_accuracy": 0.8768983954191207, + "num_tokens": 2198640.0, + "step": 1450 + }, + { + "entropy": 0.45835661113262177, + "epoch": 3.606498194945848, + "grad_norm": 0.45422980189323425, + "learning_rate": 0.0002332915140006257, + "loss": 0.40608222961425783, + "mean_token_accuracy": 0.8777871477603912, + "num_tokens": 2277538.0, + "step": 1500 + }, + { + "entropy": 0.4667848441004753, + "epoch": 3.726835138387485, + "grad_norm": 0.3852768838405609, + "learning_rate": 0.00022843383631908587, + "loss": 0.41128265380859375, + "mean_token_accuracy": 0.8771094262599946, + "num_tokens": 2355173.0, + "step": 1550 + }, + { + "entropy": 0.462973136305809, + "epoch": 3.8471720818291217, + "grad_norm": 0.5206069946289062, + "learning_rate": 0.00022342839677102398, + "loss": 0.40684612274169923, + "mean_token_accuracy": 0.8784542739391327, + "num_tokens": 2433855.0, + "step": 1600 + }, + { + "entropy": 0.47388757914304735, + "epoch": 3.967509025270758, + "grad_norm": 0.45729249715805054, + "learning_rate": 0.00021828400476169333, + "loss": 0.4176290512084961, + "mean_token_accuracy": 0.8743184685707093, + "num_tokens": 2509050.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.42192115738422054, + "eval_loss": 0.5213983058929443, + "eval_mean_token_accuracy": 0.8507144964193996, + "eval_num_tokens": 2529144.0, + "eval_runtime": 57.4742, + "eval_samples_per_second": 21.975, + "eval_steps_per_second": 2.749, + "step": 1664 + }, + { + "entropy": 0.4243522891492555, + "epoch": 4.086642599277979, + "grad_norm": 0.4242376387119293, + "learning_rate": 0.00021300971424800534, + "loss": 0.3608149719238281, + "mean_token_accuracy": 0.8896295458379418, + "num_tokens": 2584990.0, + "step": 1700 + }, + { + "entropy": 0.39899118691682817, + "epoch": 4.206979542719615, + "grad_norm": 0.527614176273346, + "learning_rate": 0.00020761480780387022, + "loss": 0.3385160446166992, + "mean_token_accuracy": 0.8941174983978272, + "num_tokens": 2660701.0, + "step": 1750 + }, + { + "entropy": 0.40566804468631745, + "epoch": 4.327316486161251, + "grad_norm": 0.4498025178909302, + "learning_rate": 0.00020210878028317902, + "loss": 0.34584579467773435, + "mean_token_accuracy": 0.8929324275255204, + "num_tokens": 2736960.0, + "step": 1800 + }, + { + "entropy": 0.4153370544314384, + "epoch": 4.447653429602888, + "grad_norm": 0.42169710993766785, + "learning_rate": 0.00019650132210918006, + "loss": 0.35420520782470705, + "mean_token_accuracy": 0.8916839396953583, + "num_tokens": 2812402.0, + "step": 1850 + }, + { + "entropy": 0.41663775861263275, + "epoch": 4.567990373044525, + "grad_norm": 0.43042847514152527, + "learning_rate": 0.00019080230221965998, + "loss": 0.3560990524291992, + "mean_token_accuracy": 0.8905349797010422, + "num_tokens": 2888713.0, + "step": 1900 + }, + { + "entropy": 0.4153156217932701, + "epoch": 4.688327316486161, + "grad_norm": 0.6334472298622131, + "learning_rate": 0.00018502175069794524, + "loss": 0.35555774688720704, + "mean_token_accuracy": 0.8908489847183227, + "num_tokens": 2966846.0, + "step": 1950 + }, + { + "entropy": 0.4233098104596138, + "epoch": 4.808664259927798, + "grad_norm": 0.4085949957370758, + "learning_rate": 0.00017916984112029274, + "loss": 0.36258663177490236, + "mean_token_accuracy": 0.8889626759290695, + "num_tokens": 3041477.0, + "step": 2000 + }, + { + "entropy": 0.41765743762254715, + "epoch": 4.929001203369435, + "grad_norm": 0.4329089820384979, + "learning_rate": 0.00017325687265073777, + "loss": 0.35529815673828125, + "mean_token_accuracy": 0.8905666440725326, + "num_tokens": 3114886.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.38224964202204836, + "eval_loss": 0.5339241027832031, + "eval_mean_token_accuracy": 0.8526725293714789, + "eval_num_tokens": 3161430.0, + "eval_runtime": 57.4581, + "eval_samples_per_second": 21.981, + "eval_steps_per_second": 2.75, + "step": 2080 + }, + { + "entropy": 0.38331641332067623, + "epoch": 5.048134777376655, + "grad_norm": 0.5011507868766785, + "learning_rate": 0.00016729325191491184, + "loss": 0.320716552734375, + "mean_token_accuracy": 0.9005015263653765, + "num_tokens": 3191728.0, + "step": 2100 + }, + { + "entropy": 0.34260627657175063, + "epoch": 5.1684717208182915, + "grad_norm": 0.5449230074882507, + "learning_rate": 0.00016128947468473123, + "loss": 0.27347917556762696, + "mean_token_accuracy": 0.9137293487787247, + "num_tokens": 3268401.0, + "step": 2150 + }, + { + "entropy": 0.33992124766111376, + "epoch": 5.2888086642599275, + "grad_norm": 0.5042031407356262, + "learning_rate": 0.0001552561074061919, + "loss": 0.2745149040222168, + "mean_token_accuracy": 0.9130180728435516, + "num_tokens": 3345743.0, + "step": 2200 + }, + { + "entropy": 0.3468990367650986, + "epoch": 5.409145607701564, + "grad_norm": 0.4478400647640228, + "learning_rate": 0.00014920376860277964, + "loss": 0.2810557746887207, + "mean_token_accuracy": 0.910719353556633, + "num_tokens": 3419875.0, + "step": 2250 + }, + { + "entropy": 0.35029833555221557, + "epoch": 5.529482551143201, + "grad_norm": 0.5645950436592102, + "learning_rate": 0.0001431431101872259, + "loss": 0.28505165100097657, + "mean_token_accuracy": 0.91094852745533, + "num_tokens": 3494189.0, + "step": 2300 + }, + { + "entropy": 0.35068276852369307, + "epoch": 5.649819494584838, + "grad_norm": 0.4856931269168854, + "learning_rate": 0.00013708479871449959, + "loss": 0.28504167556762694, + "mean_token_accuracy": 0.9086999785900116, + "num_tokens": 3571387.0, + "step": 2350 + }, + { + "entropy": 0.3502148833870888, + "epoch": 5.770156438026474, + "grad_norm": 0.4992657005786896, + "learning_rate": 0.00013103949660902948, + "loss": 0.2827724266052246, + "mean_token_accuracy": 0.9090785968303681, + "num_tokens": 3647716.0, + "step": 2400 + }, + { + "entropy": 0.34631509900093077, + "epoch": 5.890493381468111, + "grad_norm": 0.5657410621643066, + "learning_rate": 0.00012501784339919524, + "loss": 0.2854422378540039, + "mean_token_accuracy": 0.9095450508594513, + "num_tokens": 3725666.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3132856645727459, + "eval_loss": 0.6306682229042053, + "eval_mean_token_accuracy": 0.8453912821751607, + "eval_num_tokens": 3793716.0, + "eval_runtime": 57.4548, + "eval_samples_per_second": 21.983, + "eval_steps_per_second": 2.75, + "step": 2496 + }, + { + "entropy": 0.347719269569474, + "epoch": 6.009626955475331, + "grad_norm": 0.43329548835754395, + "learning_rate": 0.0001190304369921156, + "loss": 0.2800110626220703, + "mean_token_accuracy": 0.9126462815987944, + "num_tokens": 3799943.0, + "step": 2500 + }, + { + "entropy": 0.27945472195744514, + "epoch": 6.129963898916968, + "grad_norm": 0.634467601776123, + "learning_rate": 0.00011308781502168858, + "loss": 0.20203407287597655, + "mean_token_accuracy": 0.9359134542942047, + "num_tokens": 3876465.0, + "step": 2550 + }, + { + "entropy": 0.2845077550411224, + "epoch": 6.250300842358604, + "grad_norm": 0.7450694441795349, + "learning_rate": 0.00010720043630270964, + "loss": 0.2109172248840332, + "mean_token_accuracy": 0.9338040286302567, + "num_tokens": 3951466.0, + "step": 2600 + }, + { + "entropy": 0.2838969850540161, + "epoch": 6.37063778580024, + "grad_norm": 0.7350204586982727, + "learning_rate": 0.00010137866242370994, + "loss": 0.2094452476501465, + "mean_token_accuracy": 0.9322927874326706, + "num_tokens": 4026603.0, + "step": 2650 + }, + { + "entropy": 0.28683102265000343, + "epoch": 6.490974729241877, + "grad_norm": 0.6212452054023743, + "learning_rate": 9.563273951090865e-05, + "loss": 0.21383323669433593, + "mean_token_accuracy": 0.932304550409317, + "num_tokens": 4102554.0, + "step": 2700 + }, + { + "entropy": 0.27902470380067823, + "epoch": 6.611311672683514, + "grad_norm": 0.6719015836715698, + "learning_rate": 8.997278019537578e-05, + "loss": 0.2090513801574707, + "mean_token_accuracy": 0.9336883723735809, + "num_tokens": 4177827.0, + "step": 2750 + }, + { + "entropy": 0.278264344483614, + "epoch": 6.731648616125151, + "grad_norm": 0.6829904913902283, + "learning_rate": 8.440874581514121e-05, + "loss": 0.20877845764160155, + "mean_token_accuracy": 0.9345877057313919, + "num_tokens": 4253960.0, + "step": 2800 + }, + { + "entropy": 0.2799538014829159, + "epoch": 6.851985559566787, + "grad_norm": 0.71329265832901, + "learning_rate": 7.895042888357504e-05, + "loss": 0.20889324188232422, + "mean_token_accuracy": 0.933683454990387, + "num_tokens": 4332678.0, + "step": 2850 + }, + { + "entropy": 0.2825497867166996, + "epoch": 6.9723225030084235, + "grad_norm": 0.4812498688697815, + "learning_rate": 7.36074358548936e-05, + "loss": 0.21008211135864258, + "mean_token_accuracy": 0.9326896673440933, + "num_tokens": 4409204.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.26715399777587456, + "eval_loss": 0.7001190781593323, + "eval_mean_token_accuracy": 0.8474974322922623, + "eval_num_tokens": 4426002.0, + "eval_runtime": 57.4705, + "eval_samples_per_second": 21.976, + "eval_steps_per_second": 2.749, + "step": 2912 + }, + { + "entropy": 0.2501247793135017, + "epoch": 7.091456077015644, + "grad_norm": 0.46024414896965027, + "learning_rate": 6.838917021712265e-05, + "loss": 0.1688372802734375, + "mean_token_accuracy": 0.9490507395580562, + "num_tokens": 4480686.0, + "step": 2950 + }, + { + "entropy": 0.21724234893918037, + "epoch": 7.21179302045728, + "grad_norm": 0.4319011867046356, + "learning_rate": 6.330481594227512e-05, + "loss": 0.14257819175720216, + "mean_token_accuracy": 0.9567872488498688, + "num_tokens": 4559244.0, + "step": 3000 + }, + { + "entropy": 0.22447748228907585, + "epoch": 7.332129963898917, + "grad_norm": 0.6305978298187256, + "learning_rate": 5.836332132286903e-05, + "loss": 0.14830341339111328, + "mean_token_accuracy": 0.9540821850299835, + "num_tokens": 4635822.0, + "step": 3050 + }, + { + "entropy": 0.22098347604274748, + "epoch": 7.452466907340553, + "grad_norm": 0.5005761981010437, + "learning_rate": 5.3573383223233905e-05, + "loss": 0.14625223159790038, + "mean_token_accuracy": 0.9542822539806366, + "num_tokens": 4713292.0, + "step": 3100 + }, + { + "entropy": 0.2208930467069149, + "epoch": 7.57280385078219, + "grad_norm": 0.654767632484436, + "learning_rate": 4.8943431773321715e-05, + "loss": 0.14854287147521972, + "mean_token_accuracy": 0.9547992926836014, + "num_tokens": 4789589.0, + "step": 3150 + }, + { + "entropy": 0.22545789495110513, + "epoch": 7.693140794223827, + "grad_norm": 0.5283388495445251, + "learning_rate": 4.4481615531961884e-05, + "loss": 0.14947433471679689, + "mean_token_accuracy": 0.9548585432767868, + "num_tokens": 4867053.0, + "step": 3200 + }, + { + "entropy": 0.22224836826324462, + "epoch": 7.813477737665464, + "grad_norm": 0.577686071395874, + "learning_rate": 4.019578714567194e-05, + "loss": 0.14836033821105957, + "mean_token_accuracy": 0.9539271759986877, + "num_tokens": 4942970.0, + "step": 3250 + }, + { + "entropy": 0.2235196524858475, + "epoch": 7.9338146811070995, + "grad_norm": 0.47032615542411804, + "learning_rate": 3.6093489528263305e-05, + "loss": 0.14641441345214845, + "mean_token_accuracy": 0.9548134553432465, + "num_tokens": 5017197.0, + "step": 3300 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2217723396194132, + "eval_loss": 0.834998369216919, + "eval_mean_token_accuracy": 0.8450589670410639, + "eval_num_tokens": 5058288.0, + "eval_runtime": 57.4773, + "eval_samples_per_second": 21.974, + "eval_steps_per_second": 2.749, + "step": 3328 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1122808860972544e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/README.md b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/adapter_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..22fe9e5a1d9f895d47239aa10bc78cc896f11beb --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.025388885152548692, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "k_proj", + "gate_proj", + "o_proj", + "q_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/chat_template.jinja b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/tokenizer_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/trainer_state.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7adc51a80774329c6600d816a53775a270b8e549 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-3744/trainer_state.json @@ -0,0 +1,873 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3744, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.8487297534942626, + "epoch": 0.12033694344163658, + "grad_norm": 1.1108952760696411, + "learning_rate": 3.403535898928475e-05, + "loss": 2.800888977050781, + "mean_token_accuracy": 0.5476393702626229, + "num_tokens": 80236.0, + "step": 50 + }, + { + "entropy": 1.3183529460430146, + "epoch": 0.24067388688327315, + "grad_norm": 0.9604457020759583, + "learning_rate": 6.876531714161613e-05, + "loss": 1.2430918884277344, + "mean_token_accuracy": 0.7165319395065307, + "num_tokens": 158954.0, + "step": 100 + }, + { + "entropy": 0.7940413594245911, + "epoch": 0.36101083032490977, + "grad_norm": 0.6496508717536926, + "learning_rate": 0.00010349527529394752, + "loss": 0.7494664001464844, + "mean_token_accuracy": 0.7958653098344803, + "num_tokens": 235314.0, + "step": 150 + }, + { + "entropy": 0.7416071015596389, + "epoch": 0.4813477737665463, + "grad_norm": 0.7964484095573425, + "learning_rate": 0.0001382252334462789, + "loss": 0.6981658935546875, + "mean_token_accuracy": 0.8044728142023087, + "num_tokens": 311529.0, + "step": 200 + }, + { + "entropy": 0.6903449124097825, + "epoch": 0.601684717208183, + "grad_norm": 0.6642301082611084, + "learning_rate": 0.00017295519159861027, + "loss": 0.6478567504882813, + "mean_token_accuracy": 0.8178131663799286, + "num_tokens": 386903.0, + "step": 250 + }, + { + "entropy": 0.6663334459066391, + "epoch": 0.7220216606498195, + "grad_norm": 0.5385578274726868, + "learning_rate": 0.00020768514975094167, + "loss": 0.617872200012207, + "mean_token_accuracy": 0.827698912024498, + "num_tokens": 461514.0, + "step": 300 + }, + { + "entropy": 0.6437311708927155, + "epoch": 0.8423586040914561, + "grad_norm": 0.4748707413673401, + "learning_rate": 0.00024241510790327304, + "loss": 0.5984851455688477, + "mean_token_accuracy": 0.8346574056148529, + "num_tokens": 536592.0, + "step": 350 + }, + { + "entropy": 0.6274739676713943, + "epoch": 0.9626955475330926, + "grad_norm": 0.5358314514160156, + "learning_rate": 0.0002771450660556044, + "loss": 0.5809464263916015, + "mean_token_accuracy": 0.8380268800258637, + "num_tokens": 609718.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.55972887462453, + "eval_loss": 0.5503765940666199, + "eval_mean_token_accuracy": 0.8400089623807352, + "eval_num_tokens": 632286.0, + "eval_runtime": 57.4057, + "eval_samples_per_second": 22.001, + "eval_steps_per_second": 2.752, + "step": 416 + }, + { + "entropy": 0.6137794244168985, + "epoch": 1.0818291215403129, + "grad_norm": 0.48725026845932007, + "learning_rate": 0.00028889786636446425, + "loss": 0.5690413665771484, + "mean_token_accuracy": 0.8418765904927494, + "num_tokens": 683904.0, + "step": 450 + }, + { + "entropy": 0.5970469158887863, + "epoch": 1.2021660649819494, + "grad_norm": 0.5252997279167175, + "learning_rate": 0.0002886030033198732, + "loss": 0.5451018524169922, + "mean_token_accuracy": 0.8463934576511383, + "num_tokens": 758298.0, + "step": 500 + }, + { + "entropy": 0.5896219766139984, + "epoch": 1.322503008423586, + "grad_norm": 0.3888632655143738, + "learning_rate": 0.00028805448269855045, + "loss": 0.5395643997192383, + "mean_token_accuracy": 0.8473994207382202, + "num_tokens": 836432.0, + "step": 550 + }, + { + "entropy": 0.5761718067526818, + "epoch": 1.4428399518652226, + "grad_norm": 0.40175220370292664, + "learning_rate": 0.0002872532698783407, + "loss": 0.531209602355957, + "mean_token_accuracy": 0.8493033695220947, + "num_tokens": 912728.0, + "step": 600 + }, + { + "entropy": 0.5754709538817405, + "epoch": 1.5631768953068592, + "grad_norm": 0.34686869382858276, + "learning_rate": 0.0002862007749668598, + "loss": 0.529142074584961, + "mean_token_accuracy": 0.8509961807727814, + "num_tokens": 989711.0, + "step": 650 + }, + { + "entropy": 0.564461555480957, + "epoch": 1.6835138387484958, + "grad_norm": 0.3425906002521515, + "learning_rate": 0.0002848988503197527, + "loss": 0.5160605239868165, + "mean_token_accuracy": 0.8546184468269348, + "num_tokens": 1065292.0, + "step": 700 + }, + { + "entropy": 0.5677110701799393, + "epoch": 1.8038507821901324, + "grad_norm": 0.34574612975120544, + "learning_rate": 0.00028334978728060975, + "loss": 0.5171365737915039, + "mean_token_accuracy": 0.8528753453493119, + "num_tokens": 1141585.0, + "step": 750 + }, + { + "entropy": 0.5702121770381927, + "epoch": 1.924187725631769, + "grad_norm": 0.39626583456993103, + "learning_rate": 0.0002815563121482791, + "loss": 0.5194630432128906, + "mean_token_accuracy": 0.8525324720144272, + "num_tokens": 1217261.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5043281727953802, + "eval_loss": 0.5085281133651733, + "eval_mean_token_accuracy": 0.8485767777207531, + "eval_num_tokens": 1264572.0, + "eval_runtime": 57.6896, + "eval_samples_per_second": 21.893, + "eval_steps_per_second": 2.739, + "step": 832 + }, + { + "entropy": 0.5520541523442124, + "epoch": 2.0433212996389893, + "grad_norm": 0.34861499071121216, + "learning_rate": 0.00027952158137867244, + "loss": 0.4953007125854492, + "mean_token_accuracy": 0.8596346799773399, + "num_tokens": 1292148.0, + "step": 850 + }, + { + "entropy": 0.5117866912484169, + "epoch": 2.1636582430806257, + "grad_norm": 0.4384036958217621, + "learning_rate": 0.0002772491760295088, + "loss": 0.4596589660644531, + "mean_token_accuracy": 0.8644879817962646, + "num_tokens": 1368098.0, + "step": 900 + }, + { + "entropy": 0.5110387325286865, + "epoch": 2.2839951865222625, + "grad_norm": 0.44647786021232605, + "learning_rate": 0.0002747430954577733, + "loss": 0.46299396514892577, + "mean_token_accuracy": 0.8637035477161408, + "num_tokens": 1444959.0, + "step": 950 + }, + { + "entropy": 0.5270229122042656, + "epoch": 2.404332129963899, + "grad_norm": 0.3942628502845764, + "learning_rate": 0.00027200775028098345, + "loss": 0.4726287078857422, + "mean_token_accuracy": 0.8627178031206131, + "num_tokens": 1516670.0, + "step": 1000 + }, + { + "entropy": 0.518384268283844, + "epoch": 2.5246690734055353, + "grad_norm": 0.4096919298171997, + "learning_rate": 0.0002690479546146505, + "loss": 0.463576545715332, + "mean_token_accuracy": 0.863879257440567, + "num_tokens": 1592121.0, + "step": 1050 + }, + { + "entropy": 0.5208215075731277, + "epoch": 2.645006016847172, + "grad_norm": 0.32399967312812805, + "learning_rate": 0.0002658689175995985, + "loss": 0.47119388580322263, + "mean_token_accuracy": 0.8642315500974656, + "num_tokens": 1670766.0, + "step": 1100 + }, + { + "entropy": 0.5203104516863823, + "epoch": 2.765342960288809, + "grad_norm": 0.32846879959106445, + "learning_rate": 0.0002624762342340511, + "loss": 0.47053630828857423, + "mean_token_accuracy": 0.8640051782131195, + "num_tokens": 1746321.0, + "step": 1150 + }, + { + "entropy": 0.5191598236560822, + "epoch": 2.8856799037304453, + "grad_norm": 0.36446335911750793, + "learning_rate": 0.0002588758755266231, + "loss": 0.46290687561035154, + "mean_token_accuracy": 0.8656695383787155, + "num_tokens": 1823845.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.48268982862369925, + "eval_loss": 0.4935414791107178, + "eval_mean_token_accuracy": 0.8550615201268015, + "eval_num_tokens": 1896858.0, + "eval_runtime": 57.3895, + "eval_samples_per_second": 22.008, + "eval_steps_per_second": 2.753, + "step": 1248 + }, + { + "entropy": 0.5235216328592012, + "epoch": 3.0048134777376654, + "grad_norm": 0.3717617690563202, + "learning_rate": 0.00025507417798754624, + "loss": 0.46804916381835937, + "mean_token_accuracy": 0.8647771068293639, + "num_tokens": 1899333.0, + "step": 1250 + }, + { + "entropy": 0.4668200609087944, + "epoch": 3.125150421179302, + "grad_norm": 0.37964364886283875, + "learning_rate": 0.0002510778324766242, + "loss": 0.4062967300415039, + "mean_token_accuracy": 0.8777556318044663, + "num_tokens": 1974386.0, + "step": 1300 + }, + { + "entropy": 0.4731095978617668, + "epoch": 3.2454873646209386, + "grad_norm": 0.4152783453464508, + "learning_rate": 0.0002468938724275443, + "loss": 0.4166794586181641, + "mean_token_accuracy": 0.875180395245552, + "num_tokens": 2046775.0, + "step": 1350 + }, + { + "entropy": 0.47036315232515336, + "epoch": 3.3658243080625754, + "grad_norm": 0.41300854086875916, + "learning_rate": 0.0002425296614692708, + "loss": 0.41332015991210935, + "mean_token_accuracy": 0.8770327359437943, + "num_tokens": 2120417.0, + "step": 1400 + }, + { + "entropy": 0.46202471256256106, + "epoch": 3.4861612515042117, + "grad_norm": 0.353500634431839, + "learning_rate": 0.0002379928804663058, + "loss": 0.4094076156616211, + "mean_token_accuracy": 0.8768983954191207, + "num_tokens": 2198640.0, + "step": 1450 + }, + { + "entropy": 0.45835661113262177, + "epoch": 3.606498194945848, + "grad_norm": 0.45422980189323425, + "learning_rate": 0.0002332915140006257, + "loss": 0.40608222961425783, + "mean_token_accuracy": 0.8777871477603912, + "num_tokens": 2277538.0, + "step": 1500 + }, + { + "entropy": 0.4667848441004753, + "epoch": 3.726835138387485, + "grad_norm": 0.3852768838405609, + "learning_rate": 0.00022843383631908587, + "loss": 0.41128265380859375, + "mean_token_accuracy": 0.8771094262599946, + "num_tokens": 2355173.0, + "step": 1550 + }, + { + "entropy": 0.462973136305809, + "epoch": 3.8471720818291217, + "grad_norm": 0.5206069946289062, + "learning_rate": 0.00022342839677102398, + "loss": 0.40684612274169923, + "mean_token_accuracy": 0.8784542739391327, + "num_tokens": 2433855.0, + "step": 1600 + }, + { + "entropy": 0.47388757914304735, + "epoch": 3.967509025270758, + "grad_norm": 0.45729249715805054, + "learning_rate": 0.00021828400476169333, + "loss": 0.4176290512084961, + "mean_token_accuracy": 0.8743184685707093, + "num_tokens": 2509050.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.42192115738422054, + "eval_loss": 0.5213983058929443, + "eval_mean_token_accuracy": 0.8507144964193996, + "eval_num_tokens": 2529144.0, + "eval_runtime": 57.4742, + "eval_samples_per_second": 21.975, + "eval_steps_per_second": 2.749, + "step": 1664 + }, + { + "entropy": 0.4243522891492555, + "epoch": 4.086642599277979, + "grad_norm": 0.4242376387119293, + "learning_rate": 0.00021300971424800534, + "loss": 0.3608149719238281, + "mean_token_accuracy": 0.8896295458379418, + "num_tokens": 2584990.0, + "step": 1700 + }, + { + "entropy": 0.39899118691682817, + "epoch": 4.206979542719615, + "grad_norm": 0.527614176273346, + "learning_rate": 0.00020761480780387022, + "loss": 0.3385160446166992, + "mean_token_accuracy": 0.8941174983978272, + "num_tokens": 2660701.0, + "step": 1750 + }, + { + "entropy": 0.40566804468631745, + "epoch": 4.327316486161251, + "grad_norm": 0.4498025178909302, + "learning_rate": 0.00020210878028317902, + "loss": 0.34584579467773435, + "mean_token_accuracy": 0.8929324275255204, + "num_tokens": 2736960.0, + "step": 1800 + }, + { + "entropy": 0.4153370544314384, + "epoch": 4.447653429602888, + "grad_norm": 0.42169710993766785, + "learning_rate": 0.00019650132210918006, + "loss": 0.35420520782470705, + "mean_token_accuracy": 0.8916839396953583, + "num_tokens": 2812402.0, + "step": 1850 + }, + { + "entropy": 0.41663775861263275, + "epoch": 4.567990373044525, + "grad_norm": 0.43042847514152527, + "learning_rate": 0.00019080230221965998, + "loss": 0.3560990524291992, + "mean_token_accuracy": 0.8905349797010422, + "num_tokens": 2888713.0, + "step": 1900 + }, + { + "entropy": 0.4153156217932701, + "epoch": 4.688327316486161, + "grad_norm": 0.6334472298622131, + "learning_rate": 0.00018502175069794524, + "loss": 0.35555774688720704, + "mean_token_accuracy": 0.8908489847183227, + "num_tokens": 2966846.0, + "step": 1950 + }, + { + "entropy": 0.4233098104596138, + "epoch": 4.808664259927798, + "grad_norm": 0.4085949957370758, + "learning_rate": 0.00017916984112029274, + "loss": 0.36258663177490236, + "mean_token_accuracy": 0.8889626759290695, + "num_tokens": 3041477.0, + "step": 2000 + }, + { + "entropy": 0.41765743762254715, + "epoch": 4.929001203369435, + "grad_norm": 0.4329089820384979, + "learning_rate": 0.00017325687265073777, + "loss": 0.35529815673828125, + "mean_token_accuracy": 0.8905666440725326, + "num_tokens": 3114886.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.38224964202204836, + "eval_loss": 0.5339241027832031, + "eval_mean_token_accuracy": 0.8526725293714789, + "eval_num_tokens": 3161430.0, + "eval_runtime": 57.4581, + "eval_samples_per_second": 21.981, + "eval_steps_per_second": 2.75, + "step": 2080 + }, + { + "entropy": 0.38331641332067623, + "epoch": 5.048134777376655, + "grad_norm": 0.5011507868766785, + "learning_rate": 0.00016729325191491184, + "loss": 0.320716552734375, + "mean_token_accuracy": 0.9005015263653765, + "num_tokens": 3191728.0, + "step": 2100 + }, + { + "entropy": 0.34260627657175063, + "epoch": 5.1684717208182915, + "grad_norm": 0.5449230074882507, + "learning_rate": 0.00016128947468473123, + "loss": 0.27347917556762696, + "mean_token_accuracy": 0.9137293487787247, + "num_tokens": 3268401.0, + "step": 2150 + }, + { + "entropy": 0.33992124766111376, + "epoch": 5.2888086642599275, + "grad_norm": 0.5042031407356262, + "learning_rate": 0.0001552561074061919, + "loss": 0.2745149040222168, + "mean_token_accuracy": 0.9130180728435516, + "num_tokens": 3345743.0, + "step": 2200 + }, + { + "entropy": 0.3468990367650986, + "epoch": 5.409145607701564, + "grad_norm": 0.4478400647640228, + "learning_rate": 0.00014920376860277964, + "loss": 0.2810557746887207, + "mean_token_accuracy": 0.910719353556633, + "num_tokens": 3419875.0, + "step": 2250 + }, + { + "entropy": 0.35029833555221557, + "epoch": 5.529482551143201, + "grad_norm": 0.5645950436592102, + "learning_rate": 0.0001431431101872259, + "loss": 0.28505165100097657, + "mean_token_accuracy": 0.91094852745533, + "num_tokens": 3494189.0, + "step": 2300 + }, + { + "entropy": 0.35068276852369307, + "epoch": 5.649819494584838, + "grad_norm": 0.4856931269168854, + "learning_rate": 0.00013708479871449959, + "loss": 0.28504167556762694, + "mean_token_accuracy": 0.9086999785900116, + "num_tokens": 3571387.0, + "step": 2350 + }, + { + "entropy": 0.3502148833870888, + "epoch": 5.770156438026474, + "grad_norm": 0.4992657005786896, + "learning_rate": 0.00013103949660902948, + "loss": 0.2827724266052246, + "mean_token_accuracy": 0.9090785968303681, + "num_tokens": 3647716.0, + "step": 2400 + }, + { + "entropy": 0.34631509900093077, + "epoch": 5.890493381468111, + "grad_norm": 0.5657410621643066, + "learning_rate": 0.00012501784339919524, + "loss": 0.2854422378540039, + "mean_token_accuracy": 0.9095450508594513, + "num_tokens": 3725666.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3132856645727459, + "eval_loss": 0.6306682229042053, + "eval_mean_token_accuracy": 0.8453912821751607, + "eval_num_tokens": 3793716.0, + "eval_runtime": 57.4548, + "eval_samples_per_second": 21.983, + "eval_steps_per_second": 2.75, + "step": 2496 + }, + { + "entropy": 0.347719269569474, + "epoch": 6.009626955475331, + "grad_norm": 0.43329548835754395, + "learning_rate": 0.0001190304369921156, + "loss": 0.2800110626220703, + "mean_token_accuracy": 0.9126462815987944, + "num_tokens": 3799943.0, + "step": 2500 + }, + { + "entropy": 0.27945472195744514, + "epoch": 6.129963898916968, + "grad_norm": 0.634467601776123, + "learning_rate": 0.00011308781502168858, + "loss": 0.20203407287597655, + "mean_token_accuracy": 0.9359134542942047, + "num_tokens": 3876465.0, + "step": 2550 + }, + { + "entropy": 0.2845077550411224, + "epoch": 6.250300842358604, + "grad_norm": 0.7450694441795349, + "learning_rate": 0.00010720043630270964, + "loss": 0.2109172248840332, + "mean_token_accuracy": 0.9338040286302567, + "num_tokens": 3951466.0, + "step": 2600 + }, + { + "entropy": 0.2838969850540161, + "epoch": 6.37063778580024, + "grad_norm": 0.7350204586982727, + "learning_rate": 0.00010137866242370994, + "loss": 0.2094452476501465, + "mean_token_accuracy": 0.9322927874326706, + "num_tokens": 4026603.0, + "step": 2650 + }, + { + "entropy": 0.28683102265000343, + "epoch": 6.490974729241877, + "grad_norm": 0.6212452054023743, + "learning_rate": 9.563273951090865e-05, + "loss": 0.21383323669433593, + "mean_token_accuracy": 0.932304550409317, + "num_tokens": 4102554.0, + "step": 2700 + }, + { + "entropy": 0.27902470380067823, + "epoch": 6.611311672683514, + "grad_norm": 0.6719015836715698, + "learning_rate": 8.997278019537578e-05, + "loss": 0.2090513801574707, + "mean_token_accuracy": 0.9336883723735809, + "num_tokens": 4177827.0, + "step": 2750 + }, + { + "entropy": 0.278264344483614, + "epoch": 6.731648616125151, + "grad_norm": 0.6829904913902283, + "learning_rate": 8.440874581514121e-05, + "loss": 0.20877845764160155, + "mean_token_accuracy": 0.9345877057313919, + "num_tokens": 4253960.0, + "step": 2800 + }, + { + "entropy": 0.2799538014829159, + "epoch": 6.851985559566787, + "grad_norm": 0.71329265832901, + "learning_rate": 7.895042888357504e-05, + "loss": 0.20889324188232422, + "mean_token_accuracy": 0.933683454990387, + "num_tokens": 4332678.0, + "step": 2850 + }, + { + "entropy": 0.2825497867166996, + "epoch": 6.9723225030084235, + "grad_norm": 0.4812498688697815, + "learning_rate": 7.36074358548936e-05, + "loss": 0.21008211135864258, + "mean_token_accuracy": 0.9326896673440933, + "num_tokens": 4409204.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.26715399777587456, + "eval_loss": 0.7001190781593323, + "eval_mean_token_accuracy": 0.8474974322922623, + "eval_num_tokens": 4426002.0, + "eval_runtime": 57.4705, + "eval_samples_per_second": 21.976, + "eval_steps_per_second": 2.749, + "step": 2912 + }, + { + "entropy": 0.2501247793135017, + "epoch": 7.091456077015644, + "grad_norm": 0.46024414896965027, + "learning_rate": 6.838917021712265e-05, + "loss": 0.1688372802734375, + "mean_token_accuracy": 0.9490507395580562, + "num_tokens": 4480686.0, + "step": 2950 + }, + { + "entropy": 0.21724234893918037, + "epoch": 7.21179302045728, + "grad_norm": 0.4319011867046356, + "learning_rate": 6.330481594227512e-05, + "loss": 0.14257819175720216, + "mean_token_accuracy": 0.9567872488498688, + "num_tokens": 4559244.0, + "step": 3000 + }, + { + "entropy": 0.22447748228907585, + "epoch": 7.332129963898917, + "grad_norm": 0.6305978298187256, + "learning_rate": 5.836332132286903e-05, + "loss": 0.14830341339111328, + "mean_token_accuracy": 0.9540821850299835, + "num_tokens": 4635822.0, + "step": 3050 + }, + { + "entropy": 0.22098347604274748, + "epoch": 7.452466907340553, + "grad_norm": 0.5005761981010437, + "learning_rate": 5.3573383223233905e-05, + "loss": 0.14625223159790038, + "mean_token_accuracy": 0.9542822539806366, + "num_tokens": 4713292.0, + "step": 3100 + }, + { + "entropy": 0.2208930467069149, + "epoch": 7.57280385078219, + "grad_norm": 0.654767632484436, + "learning_rate": 4.8943431773321715e-05, + "loss": 0.14854287147521972, + "mean_token_accuracy": 0.9547992926836014, + "num_tokens": 4789589.0, + "step": 3150 + }, + { + "entropy": 0.22545789495110513, + "epoch": 7.693140794223827, + "grad_norm": 0.5283388495445251, + "learning_rate": 4.4481615531961884e-05, + "loss": 0.14947433471679689, + "mean_token_accuracy": 0.9548585432767868, + "num_tokens": 4867053.0, + "step": 3200 + }, + { + "entropy": 0.22224836826324462, + "epoch": 7.813477737665464, + "grad_norm": 0.577686071395874, + "learning_rate": 4.019578714567194e-05, + "loss": 0.14836033821105957, + "mean_token_accuracy": 0.9539271759986877, + "num_tokens": 4942970.0, + "step": 3250 + }, + { + "entropy": 0.2235196524858475, + "epoch": 7.9338146811070995, + "grad_norm": 0.47032615542411804, + "learning_rate": 3.6093489528263305e-05, + "loss": 0.14641441345214845, + "mean_token_accuracy": 0.9548134553432465, + "num_tokens": 5017197.0, + "step": 3300 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2217723396194132, + "eval_loss": 0.834998369216919, + "eval_mean_token_accuracy": 0.8450589670410639, + "eval_num_tokens": 5058288.0, + "eval_runtime": 57.4773, + "eval_samples_per_second": 21.974, + "eval_steps_per_second": 2.749, + "step": 3328 + }, + { + "entropy": 0.20723639368408858, + "epoch": 8.05294825511432, + "grad_norm": 0.38768312335014343, + "learning_rate": 3.218194258556732e-05, + "loss": 0.13183869361877443, + "mean_token_accuracy": 0.960036763037094, + "num_tokens": 5093176.0, + "step": 3350 + }, + { + "entropy": 0.1900019085407257, + "epoch": 8.173285198555957, + "grad_norm": 0.36849233508110046, + "learning_rate": 2.8468030508643293e-05, + "loss": 0.1157960319519043, + "mean_token_accuracy": 0.9657485383749008, + "num_tokens": 5167958.0, + "step": 3400 + }, + { + "entropy": 0.19060408160090447, + "epoch": 8.293622141997593, + "grad_norm": 0.39763137698173523, + "learning_rate": 2.4958289657833987e-05, + "loss": 0.11371180534362793, + "mean_token_accuracy": 0.9660218828916549, + "num_tokens": 5242617.0, + "step": 3450 + }, + { + "entropy": 0.19418732360005378, + "epoch": 8.41395908543923, + "grad_norm": 0.3996603190898895, + "learning_rate": 2.1658897058991303e-05, + "loss": 0.11840095520019531, + "mean_token_accuracy": 0.9639486891031265, + "num_tokens": 5315946.0, + "step": 3500 + }, + { + "entropy": 0.18080575987696648, + "epoch": 8.534296028880867, + "grad_norm": 0.42722976207733154, + "learning_rate": 1.8575659532118427e-05, + "loss": 0.1102663230895996, + "mean_token_accuracy": 0.9663107365369796, + "num_tokens": 5395540.0, + "step": 3550 + }, + { + "entropy": 0.18709433659911157, + "epoch": 8.654632972322503, + "grad_norm": 0.518858015537262, + "learning_rate": 1.5714003471561938e-05, + "loss": 0.11534343719482422, + "mean_token_accuracy": 0.9645322644710541, + "num_tokens": 5472820.0, + "step": 3600 + }, + { + "entropy": 0.18279410839080812, + "epoch": 8.774969915764139, + "grad_norm": 0.41834238171577454, + "learning_rate": 1.3078965295740338e-05, + "loss": 0.11216546058654785, + "mean_token_accuracy": 0.9659797704219818, + "num_tokens": 5550517.0, + "step": 3650 + }, + { + "entropy": 0.18989180997014046, + "epoch": 8.895306859205776, + "grad_norm": 0.5360958576202393, + "learning_rate": 1.0675182583217031e-05, + "loss": 0.11572881698608399, + "mean_token_accuracy": 0.9650873821973801, + "num_tokens": 5625607.0, + "step": 3700 + }, + { + "epoch": 9.0, + "eval_entropy": 0.19821003334054463, + "eval_loss": 0.9380669593811035, + "eval_mean_token_accuracy": 0.8448396449602102, + "eval_num_tokens": 5690574.0, + "eval_runtime": 57.4284, + "eval_samples_per_second": 21.993, + "eval_steps_per_second": 2.751, + "step": 3744 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.3760723826372608e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/README.md b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/adapter_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..22fe9e5a1d9f895d47239aa10bc78cc896f11beb --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.025388885152548692, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "k_proj", + "gate_proj", + "o_proj", + "q_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/chat_template.jinja b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/tokenizer_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/trainer_state.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..99d14620186d620b28e090d6117f4c2f8cb2a4b8 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-416/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 416, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.8487297534942626, + "epoch": 0.12033694344163658, + "grad_norm": 1.1108952760696411, + "learning_rate": 3.403535898928475e-05, + "loss": 2.800888977050781, + "mean_token_accuracy": 0.5476393702626229, + "num_tokens": 80236.0, + "step": 50 + }, + { + "entropy": 1.3183529460430146, + "epoch": 0.24067388688327315, + "grad_norm": 0.9604457020759583, + "learning_rate": 6.876531714161613e-05, + "loss": 1.2430918884277344, + "mean_token_accuracy": 0.7165319395065307, + "num_tokens": 158954.0, + "step": 100 + }, + { + "entropy": 0.7940413594245911, + "epoch": 0.36101083032490977, + "grad_norm": 0.6496508717536926, + "learning_rate": 0.00010349527529394752, + "loss": 0.7494664001464844, + "mean_token_accuracy": 0.7958653098344803, + "num_tokens": 235314.0, + "step": 150 + }, + { + "entropy": 0.7416071015596389, + "epoch": 0.4813477737665463, + "grad_norm": 0.7964484095573425, + "learning_rate": 0.0001382252334462789, + "loss": 0.6981658935546875, + "mean_token_accuracy": 0.8044728142023087, + "num_tokens": 311529.0, + "step": 200 + }, + { + "entropy": 0.6903449124097825, + "epoch": 0.601684717208183, + "grad_norm": 0.6642301082611084, + "learning_rate": 0.00017295519159861027, + "loss": 0.6478567504882813, + "mean_token_accuracy": 0.8178131663799286, + "num_tokens": 386903.0, + "step": 250 + }, + { + "entropy": 0.6663334459066391, + "epoch": 0.7220216606498195, + "grad_norm": 0.5385578274726868, + "learning_rate": 0.00020768514975094167, + "loss": 0.617872200012207, + "mean_token_accuracy": 0.827698912024498, + "num_tokens": 461514.0, + "step": 300 + }, + { + "entropy": 0.6437311708927155, + "epoch": 0.8423586040914561, + "grad_norm": 0.4748707413673401, + "learning_rate": 0.00024241510790327304, + "loss": 0.5984851455688477, + "mean_token_accuracy": 0.8346574056148529, + "num_tokens": 536592.0, + "step": 350 + }, + { + "entropy": 0.6274739676713943, + "epoch": 0.9626955475330926, + "grad_norm": 0.5358314514160156, + "learning_rate": 0.0002771450660556044, + "loss": 0.5809464263916015, + "mean_token_accuracy": 0.8380268800258637, + "num_tokens": 609718.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.55972887462453, + "eval_loss": 0.5503765940666199, + "eval_mean_token_accuracy": 0.8400089623807352, + "eval_num_tokens": 632286.0, + "eval_runtime": 57.4057, + "eval_samples_per_second": 22.001, + "eval_steps_per_second": 2.752, + "step": 416 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.651981287435776e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/README.md b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/adapter_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..22fe9e5a1d9f895d47239aa10bc78cc896f11beb --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.025388885152548692, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "k_proj", + "gate_proj", + "o_proj", + "q_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/chat_template.jinja b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/tokenizer_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/trainer_state.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..eb3ca33fb1767516dd9374b2496d59b376e7c16c --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-4160/trainer_state.json @@ -0,0 +1,974 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.8487297534942626, + "epoch": 0.12033694344163658, + "grad_norm": 1.1108952760696411, + "learning_rate": 3.403535898928475e-05, + "loss": 2.800888977050781, + "mean_token_accuracy": 0.5476393702626229, + "num_tokens": 80236.0, + "step": 50 + }, + { + "entropy": 1.3183529460430146, + "epoch": 0.24067388688327315, + "grad_norm": 0.9604457020759583, + "learning_rate": 6.876531714161613e-05, + "loss": 1.2430918884277344, + "mean_token_accuracy": 0.7165319395065307, + "num_tokens": 158954.0, + "step": 100 + }, + { + "entropy": 0.7940413594245911, + "epoch": 0.36101083032490977, + "grad_norm": 0.6496508717536926, + "learning_rate": 0.00010349527529394752, + "loss": 0.7494664001464844, + "mean_token_accuracy": 0.7958653098344803, + "num_tokens": 235314.0, + "step": 150 + }, + { + "entropy": 0.7416071015596389, + "epoch": 0.4813477737665463, + "grad_norm": 0.7964484095573425, + "learning_rate": 0.0001382252334462789, + "loss": 0.6981658935546875, + "mean_token_accuracy": 0.8044728142023087, + "num_tokens": 311529.0, + "step": 200 + }, + { + "entropy": 0.6903449124097825, + "epoch": 0.601684717208183, + "grad_norm": 0.6642301082611084, + "learning_rate": 0.00017295519159861027, + "loss": 0.6478567504882813, + "mean_token_accuracy": 0.8178131663799286, + "num_tokens": 386903.0, + "step": 250 + }, + { + "entropy": 0.6663334459066391, + "epoch": 0.7220216606498195, + "grad_norm": 0.5385578274726868, + "learning_rate": 0.00020768514975094167, + "loss": 0.617872200012207, + "mean_token_accuracy": 0.827698912024498, + "num_tokens": 461514.0, + "step": 300 + }, + { + "entropy": 0.6437311708927155, + "epoch": 0.8423586040914561, + "grad_norm": 0.4748707413673401, + "learning_rate": 0.00024241510790327304, + "loss": 0.5984851455688477, + "mean_token_accuracy": 0.8346574056148529, + "num_tokens": 536592.0, + "step": 350 + }, + { + "entropy": 0.6274739676713943, + "epoch": 0.9626955475330926, + "grad_norm": 0.5358314514160156, + "learning_rate": 0.0002771450660556044, + "loss": 0.5809464263916015, + "mean_token_accuracy": 0.8380268800258637, + "num_tokens": 609718.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.55972887462453, + "eval_loss": 0.5503765940666199, + "eval_mean_token_accuracy": 0.8400089623807352, + "eval_num_tokens": 632286.0, + "eval_runtime": 57.4057, + "eval_samples_per_second": 22.001, + "eval_steps_per_second": 2.752, + "step": 416 + }, + { + "entropy": 0.6137794244168985, + "epoch": 1.0818291215403129, + "grad_norm": 0.48725026845932007, + "learning_rate": 0.00028889786636446425, + "loss": 0.5690413665771484, + "mean_token_accuracy": 0.8418765904927494, + "num_tokens": 683904.0, + "step": 450 + }, + { + "entropy": 0.5970469158887863, + "epoch": 1.2021660649819494, + "grad_norm": 0.5252997279167175, + "learning_rate": 0.0002886030033198732, + "loss": 0.5451018524169922, + "mean_token_accuracy": 0.8463934576511383, + "num_tokens": 758298.0, + "step": 500 + }, + { + "entropy": 0.5896219766139984, + "epoch": 1.322503008423586, + "grad_norm": 0.3888632655143738, + "learning_rate": 0.00028805448269855045, + "loss": 0.5395643997192383, + "mean_token_accuracy": 0.8473994207382202, + "num_tokens": 836432.0, + "step": 550 + }, + { + "entropy": 0.5761718067526818, + "epoch": 1.4428399518652226, + "grad_norm": 0.40175220370292664, + "learning_rate": 0.0002872532698783407, + "loss": 0.531209602355957, + "mean_token_accuracy": 0.8493033695220947, + "num_tokens": 912728.0, + "step": 600 + }, + { + "entropy": 0.5754709538817405, + "epoch": 1.5631768953068592, + "grad_norm": 0.34686869382858276, + "learning_rate": 0.0002862007749668598, + "loss": 0.529142074584961, + "mean_token_accuracy": 0.8509961807727814, + "num_tokens": 989711.0, + "step": 650 + }, + { + "entropy": 0.564461555480957, + "epoch": 1.6835138387484958, + "grad_norm": 0.3425906002521515, + "learning_rate": 0.0002848988503197527, + "loss": 0.5160605239868165, + "mean_token_accuracy": 0.8546184468269348, + "num_tokens": 1065292.0, + "step": 700 + }, + { + "entropy": 0.5677110701799393, + "epoch": 1.8038507821901324, + "grad_norm": 0.34574612975120544, + "learning_rate": 0.00028334978728060975, + "loss": 0.5171365737915039, + "mean_token_accuracy": 0.8528753453493119, + "num_tokens": 1141585.0, + "step": 750 + }, + { + "entropy": 0.5702121770381927, + "epoch": 1.924187725631769, + "grad_norm": 0.39626583456993103, + "learning_rate": 0.0002815563121482791, + "loss": 0.5194630432128906, + "mean_token_accuracy": 0.8525324720144272, + "num_tokens": 1217261.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5043281727953802, + "eval_loss": 0.5085281133651733, + "eval_mean_token_accuracy": 0.8485767777207531, + "eval_num_tokens": 1264572.0, + "eval_runtime": 57.6896, + "eval_samples_per_second": 21.893, + "eval_steps_per_second": 2.739, + "step": 832 + }, + { + "entropy": 0.5520541523442124, + "epoch": 2.0433212996389893, + "grad_norm": 0.34861499071121216, + "learning_rate": 0.00027952158137867244, + "loss": 0.4953007125854492, + "mean_token_accuracy": 0.8596346799773399, + "num_tokens": 1292148.0, + "step": 850 + }, + { + "entropy": 0.5117866912484169, + "epoch": 2.1636582430806257, + "grad_norm": 0.4384036958217621, + "learning_rate": 0.0002772491760295088, + "loss": 0.4596589660644531, + "mean_token_accuracy": 0.8644879817962646, + "num_tokens": 1368098.0, + "step": 900 + }, + { + "entropy": 0.5110387325286865, + "epoch": 2.2839951865222625, + "grad_norm": 0.44647786021232605, + "learning_rate": 0.0002747430954577733, + "loss": 0.46299396514892577, + "mean_token_accuracy": 0.8637035477161408, + "num_tokens": 1444959.0, + "step": 950 + }, + { + "entropy": 0.5270229122042656, + "epoch": 2.404332129963899, + "grad_norm": 0.3942628502845764, + "learning_rate": 0.00027200775028098345, + "loss": 0.4726287078857422, + "mean_token_accuracy": 0.8627178031206131, + "num_tokens": 1516670.0, + "step": 1000 + }, + { + "entropy": 0.518384268283844, + "epoch": 2.5246690734055353, + "grad_norm": 0.4096919298171997, + "learning_rate": 0.0002690479546146505, + "loss": 0.463576545715332, + "mean_token_accuracy": 0.863879257440567, + "num_tokens": 1592121.0, + "step": 1050 + }, + { + "entropy": 0.5208215075731277, + "epoch": 2.645006016847172, + "grad_norm": 0.32399967312812805, + "learning_rate": 0.0002658689175995985, + "loss": 0.47119388580322263, + "mean_token_accuracy": 0.8642315500974656, + "num_tokens": 1670766.0, + "step": 1100 + }, + { + "entropy": 0.5203104516863823, + "epoch": 2.765342960288809, + "grad_norm": 0.32846879959106445, + "learning_rate": 0.0002624762342340511, + "loss": 0.47053630828857423, + "mean_token_accuracy": 0.8640051782131195, + "num_tokens": 1746321.0, + "step": 1150 + }, + { + "entropy": 0.5191598236560822, + "epoch": 2.8856799037304453, + "grad_norm": 0.36446335911750793, + "learning_rate": 0.0002588758755266231, + "loss": 0.46290687561035154, + "mean_token_accuracy": 0.8656695383787155, + "num_tokens": 1823845.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.48268982862369925, + "eval_loss": 0.4935414791107178, + "eval_mean_token_accuracy": 0.8550615201268015, + "eval_num_tokens": 1896858.0, + "eval_runtime": 57.3895, + "eval_samples_per_second": 22.008, + "eval_steps_per_second": 2.753, + "step": 1248 + }, + { + "entropy": 0.5235216328592012, + "epoch": 3.0048134777376654, + "grad_norm": 0.3717617690563202, + "learning_rate": 0.00025507417798754624, + "loss": 0.46804916381835937, + "mean_token_accuracy": 0.8647771068293639, + "num_tokens": 1899333.0, + "step": 1250 + }, + { + "entropy": 0.4668200609087944, + "epoch": 3.125150421179302, + "grad_norm": 0.37964364886283875, + "learning_rate": 0.0002510778324766242, + "loss": 0.4062967300415039, + "mean_token_accuracy": 0.8777556318044663, + "num_tokens": 1974386.0, + "step": 1300 + }, + { + "entropy": 0.4731095978617668, + "epoch": 3.2454873646209386, + "grad_norm": 0.4152783453464508, + "learning_rate": 0.0002468938724275443, + "loss": 0.4166794586181641, + "mean_token_accuracy": 0.875180395245552, + "num_tokens": 2046775.0, + "step": 1350 + }, + { + "entropy": 0.47036315232515336, + "epoch": 3.3658243080625754, + "grad_norm": 0.41300854086875916, + "learning_rate": 0.0002425296614692708, + "loss": 0.41332015991210935, + "mean_token_accuracy": 0.8770327359437943, + "num_tokens": 2120417.0, + "step": 1400 + }, + { + "entropy": 0.46202471256256106, + "epoch": 3.4861612515042117, + "grad_norm": 0.353500634431839, + "learning_rate": 0.0002379928804663058, + "loss": 0.4094076156616211, + "mean_token_accuracy": 0.8768983954191207, + "num_tokens": 2198640.0, + "step": 1450 + }, + { + "entropy": 0.45835661113262177, + "epoch": 3.606498194945848, + "grad_norm": 0.45422980189323425, + "learning_rate": 0.0002332915140006257, + "loss": 0.40608222961425783, + "mean_token_accuracy": 0.8777871477603912, + "num_tokens": 2277538.0, + "step": 1500 + }, + { + "entropy": 0.4667848441004753, + "epoch": 3.726835138387485, + "grad_norm": 0.3852768838405609, + "learning_rate": 0.00022843383631908587, + "loss": 0.41128265380859375, + "mean_token_accuracy": 0.8771094262599946, + "num_tokens": 2355173.0, + "step": 1550 + }, + { + "entropy": 0.462973136305809, + "epoch": 3.8471720818291217, + "grad_norm": 0.5206069946289062, + "learning_rate": 0.00022342839677102398, + "loss": 0.40684612274169923, + "mean_token_accuracy": 0.8784542739391327, + "num_tokens": 2433855.0, + "step": 1600 + }, + { + "entropy": 0.47388757914304735, + "epoch": 3.967509025270758, + "grad_norm": 0.45729249715805054, + "learning_rate": 0.00021828400476169333, + "loss": 0.4176290512084961, + "mean_token_accuracy": 0.8743184685707093, + "num_tokens": 2509050.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.42192115738422054, + "eval_loss": 0.5213983058929443, + "eval_mean_token_accuracy": 0.8507144964193996, + "eval_num_tokens": 2529144.0, + "eval_runtime": 57.4742, + "eval_samples_per_second": 21.975, + "eval_steps_per_second": 2.749, + "step": 1664 + }, + { + "entropy": 0.4243522891492555, + "epoch": 4.086642599277979, + "grad_norm": 0.4242376387119293, + "learning_rate": 0.00021300971424800534, + "loss": 0.3608149719238281, + "mean_token_accuracy": 0.8896295458379418, + "num_tokens": 2584990.0, + "step": 1700 + }, + { + "entropy": 0.39899118691682817, + "epoch": 4.206979542719615, + "grad_norm": 0.527614176273346, + "learning_rate": 0.00020761480780387022, + "loss": 0.3385160446166992, + "mean_token_accuracy": 0.8941174983978272, + "num_tokens": 2660701.0, + "step": 1750 + }, + { + "entropy": 0.40566804468631745, + "epoch": 4.327316486161251, + "grad_norm": 0.4498025178909302, + "learning_rate": 0.00020210878028317902, + "loss": 0.34584579467773435, + "mean_token_accuracy": 0.8929324275255204, + "num_tokens": 2736960.0, + "step": 1800 + }, + { + "entropy": 0.4153370544314384, + "epoch": 4.447653429602888, + "grad_norm": 0.42169710993766785, + "learning_rate": 0.00019650132210918006, + "loss": 0.35420520782470705, + "mean_token_accuracy": 0.8916839396953583, + "num_tokens": 2812402.0, + "step": 1850 + }, + { + "entropy": 0.41663775861263275, + "epoch": 4.567990373044525, + "grad_norm": 0.43042847514152527, + "learning_rate": 0.00019080230221965998, + "loss": 0.3560990524291992, + "mean_token_accuracy": 0.8905349797010422, + "num_tokens": 2888713.0, + "step": 1900 + }, + { + "entropy": 0.4153156217932701, + "epoch": 4.688327316486161, + "grad_norm": 0.6334472298622131, + "learning_rate": 0.00018502175069794524, + "loss": 0.35555774688720704, + "mean_token_accuracy": 0.8908489847183227, + "num_tokens": 2966846.0, + "step": 1950 + }, + { + "entropy": 0.4233098104596138, + "epoch": 4.808664259927798, + "grad_norm": 0.4085949957370758, + "learning_rate": 0.00017916984112029274, + "loss": 0.36258663177490236, + "mean_token_accuracy": 0.8889626759290695, + "num_tokens": 3041477.0, + "step": 2000 + }, + { + "entropy": 0.41765743762254715, + "epoch": 4.929001203369435, + "grad_norm": 0.4329089820384979, + "learning_rate": 0.00017325687265073777, + "loss": 0.35529815673828125, + "mean_token_accuracy": 0.8905666440725326, + "num_tokens": 3114886.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.38224964202204836, + "eval_loss": 0.5339241027832031, + "eval_mean_token_accuracy": 0.8526725293714789, + "eval_num_tokens": 3161430.0, + "eval_runtime": 57.4581, + "eval_samples_per_second": 21.981, + "eval_steps_per_second": 2.75, + "step": 2080 + }, + { + "entropy": 0.38331641332067623, + "epoch": 5.048134777376655, + "grad_norm": 0.5011507868766785, + "learning_rate": 0.00016729325191491184, + "loss": 0.320716552734375, + "mean_token_accuracy": 0.9005015263653765, + "num_tokens": 3191728.0, + "step": 2100 + }, + { + "entropy": 0.34260627657175063, + "epoch": 5.1684717208182915, + "grad_norm": 0.5449230074882507, + "learning_rate": 0.00016128947468473123, + "loss": 0.27347917556762696, + "mean_token_accuracy": 0.9137293487787247, + "num_tokens": 3268401.0, + "step": 2150 + }, + { + "entropy": 0.33992124766111376, + "epoch": 5.2888086642599275, + "grad_norm": 0.5042031407356262, + "learning_rate": 0.0001552561074061919, + "loss": 0.2745149040222168, + "mean_token_accuracy": 0.9130180728435516, + "num_tokens": 3345743.0, + "step": 2200 + }, + { + "entropy": 0.3468990367650986, + "epoch": 5.409145607701564, + "grad_norm": 0.4478400647640228, + "learning_rate": 0.00014920376860277964, + "loss": 0.2810557746887207, + "mean_token_accuracy": 0.910719353556633, + "num_tokens": 3419875.0, + "step": 2250 + }, + { + "entropy": 0.35029833555221557, + "epoch": 5.529482551143201, + "grad_norm": 0.5645950436592102, + "learning_rate": 0.0001431431101872259, + "loss": 0.28505165100097657, + "mean_token_accuracy": 0.91094852745533, + "num_tokens": 3494189.0, + "step": 2300 + }, + { + "entropy": 0.35068276852369307, + "epoch": 5.649819494584838, + "grad_norm": 0.4856931269168854, + "learning_rate": 0.00013708479871449959, + "loss": 0.28504167556762694, + "mean_token_accuracy": 0.9086999785900116, + "num_tokens": 3571387.0, + "step": 2350 + }, + { + "entropy": 0.3502148833870888, + "epoch": 5.770156438026474, + "grad_norm": 0.4992657005786896, + "learning_rate": 0.00013103949660902948, + "loss": 0.2827724266052246, + "mean_token_accuracy": 0.9090785968303681, + "num_tokens": 3647716.0, + "step": 2400 + }, + { + "entropy": 0.34631509900093077, + "epoch": 5.890493381468111, + "grad_norm": 0.5657410621643066, + "learning_rate": 0.00012501784339919524, + "loss": 0.2854422378540039, + "mean_token_accuracy": 0.9095450508594513, + "num_tokens": 3725666.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3132856645727459, + "eval_loss": 0.6306682229042053, + "eval_mean_token_accuracy": 0.8453912821751607, + "eval_num_tokens": 3793716.0, + "eval_runtime": 57.4548, + "eval_samples_per_second": 21.983, + "eval_steps_per_second": 2.75, + "step": 2496 + }, + { + "entropy": 0.347719269569474, + "epoch": 6.009626955475331, + "grad_norm": 0.43329548835754395, + "learning_rate": 0.0001190304369921156, + "loss": 0.2800110626220703, + "mean_token_accuracy": 0.9126462815987944, + "num_tokens": 3799943.0, + "step": 2500 + }, + { + "entropy": 0.27945472195744514, + "epoch": 6.129963898916968, + "grad_norm": 0.634467601776123, + "learning_rate": 0.00011308781502168858, + "loss": 0.20203407287597655, + "mean_token_accuracy": 0.9359134542942047, + "num_tokens": 3876465.0, + "step": 2550 + }, + { + "entropy": 0.2845077550411224, + "epoch": 6.250300842358604, + "grad_norm": 0.7450694441795349, + "learning_rate": 0.00010720043630270964, + "loss": 0.2109172248840332, + "mean_token_accuracy": 0.9338040286302567, + "num_tokens": 3951466.0, + "step": 2600 + }, + { + "entropy": 0.2838969850540161, + "epoch": 6.37063778580024, + "grad_norm": 0.7350204586982727, + "learning_rate": 0.00010137866242370994, + "loss": 0.2094452476501465, + "mean_token_accuracy": 0.9322927874326706, + "num_tokens": 4026603.0, + "step": 2650 + }, + { + "entropy": 0.28683102265000343, + "epoch": 6.490974729241877, + "grad_norm": 0.6212452054023743, + "learning_rate": 9.563273951090865e-05, + "loss": 0.21383323669433593, + "mean_token_accuracy": 0.932304550409317, + "num_tokens": 4102554.0, + "step": 2700 + }, + { + "entropy": 0.27902470380067823, + "epoch": 6.611311672683514, + "grad_norm": 0.6719015836715698, + "learning_rate": 8.997278019537578e-05, + "loss": 0.2090513801574707, + "mean_token_accuracy": 0.9336883723735809, + "num_tokens": 4177827.0, + "step": 2750 + }, + { + "entropy": 0.278264344483614, + "epoch": 6.731648616125151, + "grad_norm": 0.6829904913902283, + "learning_rate": 8.440874581514121e-05, + "loss": 0.20877845764160155, + "mean_token_accuracy": 0.9345877057313919, + "num_tokens": 4253960.0, + "step": 2800 + }, + { + "entropy": 0.2799538014829159, + "epoch": 6.851985559566787, + "grad_norm": 0.71329265832901, + "learning_rate": 7.895042888357504e-05, + "loss": 0.20889324188232422, + "mean_token_accuracy": 0.933683454990387, + "num_tokens": 4332678.0, + "step": 2850 + }, + { + "entropy": 0.2825497867166996, + "epoch": 6.9723225030084235, + "grad_norm": 0.4812498688697815, + "learning_rate": 7.36074358548936e-05, + "loss": 0.21008211135864258, + "mean_token_accuracy": 0.9326896673440933, + "num_tokens": 4409204.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.26715399777587456, + "eval_loss": 0.7001190781593323, + "eval_mean_token_accuracy": 0.8474974322922623, + "eval_num_tokens": 4426002.0, + "eval_runtime": 57.4705, + "eval_samples_per_second": 21.976, + "eval_steps_per_second": 2.749, + "step": 2912 + }, + { + "entropy": 0.2501247793135017, + "epoch": 7.091456077015644, + "grad_norm": 0.46024414896965027, + "learning_rate": 6.838917021712265e-05, + "loss": 0.1688372802734375, + "mean_token_accuracy": 0.9490507395580562, + "num_tokens": 4480686.0, + "step": 2950 + }, + { + "entropy": 0.21724234893918037, + "epoch": 7.21179302045728, + "grad_norm": 0.4319011867046356, + "learning_rate": 6.330481594227512e-05, + "loss": 0.14257819175720216, + "mean_token_accuracy": 0.9567872488498688, + "num_tokens": 4559244.0, + "step": 3000 + }, + { + "entropy": 0.22447748228907585, + "epoch": 7.332129963898917, + "grad_norm": 0.6305978298187256, + "learning_rate": 5.836332132286903e-05, + "loss": 0.14830341339111328, + "mean_token_accuracy": 0.9540821850299835, + "num_tokens": 4635822.0, + "step": 3050 + }, + { + "entropy": 0.22098347604274748, + "epoch": 7.452466907340553, + "grad_norm": 0.5005761981010437, + "learning_rate": 5.3573383223233905e-05, + "loss": 0.14625223159790038, + "mean_token_accuracy": 0.9542822539806366, + "num_tokens": 4713292.0, + "step": 3100 + }, + { + "entropy": 0.2208930467069149, + "epoch": 7.57280385078219, + "grad_norm": 0.654767632484436, + "learning_rate": 4.8943431773321715e-05, + "loss": 0.14854287147521972, + "mean_token_accuracy": 0.9547992926836014, + "num_tokens": 4789589.0, + "step": 3150 + }, + { + "entropy": 0.22545789495110513, + "epoch": 7.693140794223827, + "grad_norm": 0.5283388495445251, + "learning_rate": 4.4481615531961884e-05, + "loss": 0.14947433471679689, + "mean_token_accuracy": 0.9548585432767868, + "num_tokens": 4867053.0, + "step": 3200 + }, + { + "entropy": 0.22224836826324462, + "epoch": 7.813477737665464, + "grad_norm": 0.577686071395874, + "learning_rate": 4.019578714567194e-05, + "loss": 0.14836033821105957, + "mean_token_accuracy": 0.9539271759986877, + "num_tokens": 4942970.0, + "step": 3250 + }, + { + "entropy": 0.2235196524858475, + "epoch": 7.9338146811070995, + "grad_norm": 0.47032615542411804, + "learning_rate": 3.6093489528263305e-05, + "loss": 0.14641441345214845, + "mean_token_accuracy": 0.9548134553432465, + "num_tokens": 5017197.0, + "step": 3300 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2217723396194132, + "eval_loss": 0.834998369216919, + "eval_mean_token_accuracy": 0.8450589670410639, + "eval_num_tokens": 5058288.0, + "eval_runtime": 57.4773, + "eval_samples_per_second": 21.974, + "eval_steps_per_second": 2.749, + "step": 3328 + }, + { + "entropy": 0.20723639368408858, + "epoch": 8.05294825511432, + "grad_norm": 0.38768312335014343, + "learning_rate": 3.218194258556732e-05, + "loss": 0.13183869361877443, + "mean_token_accuracy": 0.960036763037094, + "num_tokens": 5093176.0, + "step": 3350 + }, + { + "entropy": 0.1900019085407257, + "epoch": 8.173285198555957, + "grad_norm": 0.36849233508110046, + "learning_rate": 2.8468030508643293e-05, + "loss": 0.1157960319519043, + "mean_token_accuracy": 0.9657485383749008, + "num_tokens": 5167958.0, + "step": 3400 + }, + { + "entropy": 0.19060408160090447, + "epoch": 8.293622141997593, + "grad_norm": 0.39763137698173523, + "learning_rate": 2.4958289657833987e-05, + "loss": 0.11371180534362793, + "mean_token_accuracy": 0.9660218828916549, + "num_tokens": 5242617.0, + "step": 3450 + }, + { + "entropy": 0.19418732360005378, + "epoch": 8.41395908543923, + "grad_norm": 0.3996603190898895, + "learning_rate": 2.1658897058991303e-05, + "loss": 0.11840095520019531, + "mean_token_accuracy": 0.9639486891031265, + "num_tokens": 5315946.0, + "step": 3500 + }, + { + "entropy": 0.18080575987696648, + "epoch": 8.534296028880867, + "grad_norm": 0.42722976207733154, + "learning_rate": 1.8575659532118427e-05, + "loss": 0.1102663230895996, + "mean_token_accuracy": 0.9663107365369796, + "num_tokens": 5395540.0, + "step": 3550 + }, + { + "entropy": 0.18709433659911157, + "epoch": 8.654632972322503, + "grad_norm": 0.518858015537262, + "learning_rate": 1.5714003471561938e-05, + "loss": 0.11534343719482422, + "mean_token_accuracy": 0.9645322644710541, + "num_tokens": 5472820.0, + "step": 3600 + }, + { + "entropy": 0.18279410839080812, + "epoch": 8.774969915764139, + "grad_norm": 0.41834238171577454, + "learning_rate": 1.3078965295740338e-05, + "loss": 0.11216546058654785, + "mean_token_accuracy": 0.9659797704219818, + "num_tokens": 5550517.0, + "step": 3650 + }, + { + "entropy": 0.18989180997014046, + "epoch": 8.895306859205776, + "grad_norm": 0.5360958576202393, + "learning_rate": 1.0675182583217031e-05, + "loss": 0.11572881698608399, + "mean_token_accuracy": 0.9650873821973801, + "num_tokens": 5625607.0, + "step": 3700 + }, + { + "epoch": 9.0, + "eval_entropy": 0.19821003334054463, + "eval_loss": 0.9380669593811035, + "eval_mean_token_accuracy": 0.8448396449602102, + "eval_num_tokens": 5690574.0, + "eval_runtime": 57.4284, + "eval_samples_per_second": 21.993, + "eval_steps_per_second": 2.751, + "step": 3744 + }, + { + "entropy": 0.18476941248383186, + "epoch": 9.014440433212997, + "grad_norm": 0.25600916147232056, + "learning_rate": 8.50688591071794e-06, + "loss": 0.11299485206604004, + "mean_token_accuracy": 0.9663846872069619, + "num_tokens": 5700391.0, + "step": 3750 + }, + { + "entropy": 0.1755327522754669, + "epoch": 9.134777376654633, + "grad_norm": 0.31042489409446716, + "learning_rate": 6.577891407458794e-06, + "loss": 0.10061668395996094, + "mean_token_accuracy": 0.9704844796657562, + "num_tokens": 5778893.0, + "step": 3800 + }, + { + "entropy": 0.17378679774701594, + "epoch": 9.25511432009627, + "grad_norm": 0.40692880749702454, + "learning_rate": 4.891594038886159e-06, + "loss": 0.09997118949890137, + "mean_token_accuracy": 0.970051052570343, + "num_tokens": 5856885.0, + "step": 3850 + }, + { + "entropy": 0.1763426448404789, + "epoch": 9.375451263537906, + "grad_norm": 0.36374372243881226, + "learning_rate": 3.4509616316522585e-06, + "loss": 0.10002938270568848, + "mean_token_accuracy": 0.9698185837268829, + "num_tokens": 5933827.0, + "step": 3900 + }, + { + "entropy": 0.17935243897140027, + "epoch": 9.495788206979542, + "grad_norm": 0.3686467707157135, + "learning_rate": 2.2585296503400713e-06, + "loss": 0.10379327774047852, + "mean_token_accuracy": 0.9684068405628204, + "num_tokens": 6009198.0, + "step": 3950 + }, + { + "entropy": 0.17841301068663598, + "epoch": 9.61612515042118, + "grad_norm": 0.4648931920528412, + "learning_rate": 1.3163967351307742e-06, + "loss": 0.10575074195861817, + "mean_token_accuracy": 0.9682148271799087, + "num_tokens": 6083601.0, + "step": 4000 + }, + { + "entropy": 0.17624000295996667, + "epoch": 9.736462093862816, + "grad_norm": 0.43090373277664185, + "learning_rate": 6.262210082677343e-07, + "loss": 0.10240504264831543, + "mean_token_accuracy": 0.9687591433525086, + "num_tokens": 6160049.0, + "step": 4050 + }, + { + "entropy": 0.18228888489305972, + "epoch": 9.856799037304452, + "grad_norm": 0.3078688681125641, + "learning_rate": 1.8921715581707156e-07, + "loss": 0.10593132972717285, + "mean_token_accuracy": 0.9678287827968597, + "num_tokens": 6233799.0, + "step": 4100 + }, + { + "entropy": 0.1741529805958271, + "epoch": 9.97713598074609, + "grad_norm": 0.4327860176563263, + "learning_rate": 6.154289861034489e-09, + "loss": 0.10117217063903809, + "mean_token_accuracy": 0.9692896944284439, + "num_tokens": 6309872.0, + "step": 4150 + }, + { + "epoch": 10.0, + "eval_entropy": 0.1901972328748884, + "eval_loss": 0.9967633485794067, + "eval_mean_token_accuracy": 0.8438419239430488, + "eval_num_tokens": 6322860.0, + "eval_runtime": 57.4912, + "eval_samples_per_second": 21.969, + "eval_steps_per_second": 2.748, + "step": 4160 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.6392298607207936e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/README.md b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab1c84818e69f811f16f60142e1cc06e6613121 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/adapter_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..22fe9e5a1d9f895d47239aa10bc78cc896f11beb --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.025388885152548692, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "k_proj", + "gate_proj", + "o_proj", + "q_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/chat_template.jinja b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..699ff8df401fe4788525e9c1f9b86a99eadd6230 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/tokenizer_config.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c960ecf0d33fd7b8c99d12680c0e74a82b36d446 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/trainer_state.json b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b1042f803033217eaa313f7f6513c155bf5ebe76 --- /dev/null +++ b/productivity_code_Swedish/Qwen3-4B-Base_productivity_splits_code_features_train_productivity_splits_code_features_test1/checkpoint-832/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 832, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.8487297534942626, + "epoch": 0.12033694344163658, + "grad_norm": 1.1108952760696411, + "learning_rate": 3.403535898928475e-05, + "loss": 2.800888977050781, + "mean_token_accuracy": 0.5476393702626229, + "num_tokens": 80236.0, + "step": 50 + }, + { + "entropy": 1.3183529460430146, + "epoch": 0.24067388688327315, + "grad_norm": 0.9604457020759583, + "learning_rate": 6.876531714161613e-05, + "loss": 1.2430918884277344, + "mean_token_accuracy": 0.7165319395065307, + "num_tokens": 158954.0, + "step": 100 + }, + { + "entropy": 0.7940413594245911, + "epoch": 0.36101083032490977, + "grad_norm": 0.6496508717536926, + "learning_rate": 0.00010349527529394752, + "loss": 0.7494664001464844, + "mean_token_accuracy": 0.7958653098344803, + "num_tokens": 235314.0, + "step": 150 + }, + { + "entropy": 0.7416071015596389, + "epoch": 0.4813477737665463, + "grad_norm": 0.7964484095573425, + "learning_rate": 0.0001382252334462789, + "loss": 0.6981658935546875, + "mean_token_accuracy": 0.8044728142023087, + "num_tokens": 311529.0, + "step": 200 + }, + { + "entropy": 0.6903449124097825, + "epoch": 0.601684717208183, + "grad_norm": 0.6642301082611084, + "learning_rate": 0.00017295519159861027, + "loss": 0.6478567504882813, + "mean_token_accuracy": 0.8178131663799286, + "num_tokens": 386903.0, + "step": 250 + }, + { + "entropy": 0.6663334459066391, + "epoch": 0.7220216606498195, + "grad_norm": 0.5385578274726868, + "learning_rate": 0.00020768514975094167, + "loss": 0.617872200012207, + "mean_token_accuracy": 0.827698912024498, + "num_tokens": 461514.0, + "step": 300 + }, + { + "entropy": 0.6437311708927155, + "epoch": 0.8423586040914561, + "grad_norm": 0.4748707413673401, + "learning_rate": 0.00024241510790327304, + "loss": 0.5984851455688477, + "mean_token_accuracy": 0.8346574056148529, + "num_tokens": 536592.0, + "step": 350 + }, + { + "entropy": 0.6274739676713943, + "epoch": 0.9626955475330926, + "grad_norm": 0.5358314514160156, + "learning_rate": 0.0002771450660556044, + "loss": 0.5809464263916015, + "mean_token_accuracy": 0.8380268800258637, + "num_tokens": 609718.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.55972887462453, + "eval_loss": 0.5503765940666199, + "eval_mean_token_accuracy": 0.8400089623807352, + "eval_num_tokens": 632286.0, + "eval_runtime": 57.4057, + "eval_samples_per_second": 22.001, + "eval_steps_per_second": 2.752, + "step": 416 + }, + { + "entropy": 0.6137794244168985, + "epoch": 1.0818291215403129, + "grad_norm": 0.48725026845932007, + "learning_rate": 0.00028889786636446425, + "loss": 0.5690413665771484, + "mean_token_accuracy": 0.8418765904927494, + "num_tokens": 683904.0, + "step": 450 + }, + { + "entropy": 0.5970469158887863, + "epoch": 1.2021660649819494, + "grad_norm": 0.5252997279167175, + "learning_rate": 0.0002886030033198732, + "loss": 0.5451018524169922, + "mean_token_accuracy": 0.8463934576511383, + "num_tokens": 758298.0, + "step": 500 + }, + { + "entropy": 0.5896219766139984, + "epoch": 1.322503008423586, + "grad_norm": 0.3888632655143738, + "learning_rate": 0.00028805448269855045, + "loss": 0.5395643997192383, + "mean_token_accuracy": 0.8473994207382202, + "num_tokens": 836432.0, + "step": 550 + }, + { + "entropy": 0.5761718067526818, + "epoch": 1.4428399518652226, + "grad_norm": 0.40175220370292664, + "learning_rate": 0.0002872532698783407, + "loss": 0.531209602355957, + "mean_token_accuracy": 0.8493033695220947, + "num_tokens": 912728.0, + "step": 600 + }, + { + "entropy": 0.5754709538817405, + "epoch": 1.5631768953068592, + "grad_norm": 0.34686869382858276, + "learning_rate": 0.0002862007749668598, + "loss": 0.529142074584961, + "mean_token_accuracy": 0.8509961807727814, + "num_tokens": 989711.0, + "step": 650 + }, + { + "entropy": 0.564461555480957, + "epoch": 1.6835138387484958, + "grad_norm": 0.3425906002521515, + "learning_rate": 0.0002848988503197527, + "loss": 0.5160605239868165, + "mean_token_accuracy": 0.8546184468269348, + "num_tokens": 1065292.0, + "step": 700 + }, + { + "entropy": 0.5677110701799393, + "epoch": 1.8038507821901324, + "grad_norm": 0.34574612975120544, + "learning_rate": 0.00028334978728060975, + "loss": 0.5171365737915039, + "mean_token_accuracy": 0.8528753453493119, + "num_tokens": 1141585.0, + "step": 750 + }, + { + "entropy": 0.5702121770381927, + "epoch": 1.924187725631769, + "grad_norm": 0.39626583456993103, + "learning_rate": 0.0002815563121482791, + "loss": 0.5194630432128906, + "mean_token_accuracy": 0.8525324720144272, + "num_tokens": 1217261.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5043281727953802, + "eval_loss": 0.5085281133651733, + "eval_mean_token_accuracy": 0.8485767777207531, + "eval_num_tokens": 1264572.0, + "eval_runtime": 57.6896, + "eval_samples_per_second": 21.893, + "eval_steps_per_second": 2.739, + "step": 832 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.279156217775104e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}