diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..59c54f748174f4e984712590d684373f7a9505b9 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: transformers +model_name: Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/9ofk4atd) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..915ad76760dee641ab750b62daaa3f74153ec2c1 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: transformers +model_name: Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/7hobmaco) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/README.md b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/adapter_config.json b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f34a9da6fde155c2a6d8fec9848a329fa629c1a0 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.04879530666124931, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "gate_proj", + "q_proj", + "k_proj", + "down_proj", + "up_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/chat_template.jinja b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/tokenizer_config.json b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/trainer_state.json b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7cee5b94ece8468964d39275c3976c96d9d1d15d --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1245/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1245, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.1012717413902284, + "epoch": 0.12070006035003017, + "grad_norm": 1.2028698921203613, + "learning_rate": 3.313127220682932e-05, + "loss": 1.9106788635253906, + "mean_token_accuracy": 0.6147816297411919, + "num_tokens": 117421.0, + "step": 50 + }, + { + "entropy": 1.0568797352910042, + "epoch": 0.24140012070006034, + "grad_norm": 0.707794725894928, + "learning_rate": 6.693869282604292e-05, + "loss": 0.9455691528320312, + "mean_token_accuracy": 0.7514659878611565, + "num_tokens": 235894.0, + "step": 100 + }, + { + "entropy": 0.8912339144945145, + "epoch": 0.3621001810500905, + "grad_norm": 0.6819370985031128, + "learning_rate": 0.0001007461134452565, + "loss": 0.7901513671875, + "mean_token_accuracy": 0.7814472052454948, + "num_tokens": 354448.0, + "step": 150 + }, + { + "entropy": 0.8389395973086358, + "epoch": 0.4828002414001207, + "grad_norm": 0.6772453188896179, + "learning_rate": 0.0001345535340644701, + "loss": 0.7409224700927735, + "mean_token_accuracy": 0.7919744056463242, + "num_tokens": 472123.0, + "step": 200 + }, + { + "entropy": 0.818748975098133, + "epoch": 0.6035003017501509, + "grad_norm": 0.7662261724472046, + "learning_rate": 0.0001683609546836837, + "loss": 0.7154017639160156, + "mean_token_accuracy": 0.7969402438402176, + "num_tokens": 583033.0, + "step": 250 + }, + { + "entropy": 0.7894713908433915, + "epoch": 0.724200362100181, + "grad_norm": 0.6039052605628967, + "learning_rate": 0.0002021683753028973, + "loss": 0.6953791046142578, + "mean_token_accuracy": 0.8036386013031006, + "num_tokens": 702243.0, + "step": 300 + }, + { + "entropy": 0.7873514747619629, + "epoch": 0.8449004224502112, + "grad_norm": 0.5863620042800903, + "learning_rate": 0.00023597579592211088, + "loss": 0.6996137237548828, + "mean_token_accuracy": 0.8034878405928612, + "num_tokens": 818352.0, + "step": 350 + }, + { + "entropy": 0.7764339351654053, + "epoch": 0.9656004828002414, + "grad_norm": 0.49791479110717773, + "learning_rate": 0.0002697832165413245, + "loss": 0.6787944793701172, + "mean_token_accuracy": 0.8066928315162659, + "num_tokens": 926964.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7318378298470144, + "eval_loss": 0.6790198683738708, + "eval_mean_token_accuracy": 0.8080558733323987, + "eval_num_tokens": 960569.0, + "eval_runtime": 88.4043, + "eval_samples_per_second": 16.063, + "eval_steps_per_second": 2.013, + "step": 415 + }, + { + "entropy": 0.7384049266122924, + "epoch": 1.0844900422450212, + "grad_norm": 0.5788671970367432, + "learning_rate": 0.0002805442221888677, + "loss": 0.6423238372802734, + "mean_token_accuracy": 0.8125922885643044, + "num_tokens": 1044171.0, + "step": 450 + }, + { + "entropy": 0.7178834271430969, + "epoch": 1.2051901025950513, + "grad_norm": 0.5945919752120972, + "learning_rate": 0.00028025154397167963, + "loss": 0.628162841796875, + "mean_token_accuracy": 0.8169734939932823, + "num_tokens": 1161114.0, + "step": 500 + }, + { + "entropy": 0.7209221491217613, + "epoch": 1.3258901629450814, + "grad_norm": 0.49619191884994507, + "learning_rate": 0.000279711368859939, + "loss": 0.6290118408203125, + "mean_token_accuracy": 0.8175607281923294, + "num_tokens": 1278659.0, + "step": 550 + }, + { + "entropy": 0.7152036108076573, + "epoch": 1.4465902232951118, + "grad_norm": 0.5819481015205383, + "learning_rate": 0.00027892465213015695, + "loss": 0.6237701416015625, + "mean_token_accuracy": 0.8183089861273766, + "num_tokens": 1388746.0, + "step": 600 + }, + { + "entropy": 0.706106402873993, + "epoch": 1.567290283645142, + "grad_norm": 0.5964105129241943, + "learning_rate": 0.0002778927850571194, + "loss": 0.6117628479003906, + "mean_token_accuracy": 0.8191776639223098, + "num_tokens": 1503454.0, + "step": 650 + }, + { + "entropy": 0.6981569758057594, + "epoch": 1.687990343995172, + "grad_norm": 0.4185747504234314, + "learning_rate": 0.0002766175924534771, + "loss": 0.6114554214477539, + "mean_token_accuracy": 0.8221894565224648, + "num_tokens": 1620874.0, + "step": 700 + }, + { + "entropy": 0.6924624507129192, + "epoch": 1.8086904043452021, + "grad_norm": 0.5334014296531677, + "learning_rate": 0.0002751013294426428, + "loss": 0.6072402572631836, + "mean_token_accuracy": 0.8222330984473228, + "num_tokens": 1742527.0, + "step": 750 + }, + { + "entropy": 0.6838067446649074, + "epoch": 1.9293904646952322, + "grad_norm": 0.4910935163497925, + "learning_rate": 0.0002733466774707023, + "loss": 0.5899914169311523, + "mean_token_accuracy": 0.8244012910127639, + "num_tokens": 1856624.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6870943672871321, + "eval_loss": 0.6236395835876465, + "eval_mean_token_accuracy": 0.8221416041422426, + "eval_num_tokens": 1921138.0, + "eval_runtime": 89.423, + "eval_samples_per_second": 15.88, + "eval_steps_per_second": 1.991, + "step": 830 + }, + { + "entropy": 0.6511646548503547, + "epoch": 2.048280024140012, + "grad_norm": 0.3761918246746063, + "learning_rate": 0.0002713567395643921, + "loss": 0.5689863586425781, + "mean_token_accuracy": 0.830169784538637, + "num_tokens": 1971453.0, + "step": 850 + }, + { + "entropy": 0.6192687456309796, + "epoch": 2.1689800844900424, + "grad_norm": 0.5916118025779724, + "learning_rate": 0.0002691350348435296, + "loss": 0.5307585906982422, + "mean_token_accuracy": 0.8394980162382126, + "num_tokens": 2089506.0, + "step": 900 + }, + { + "entropy": 0.6199816593527794, + "epoch": 2.2896801448400725, + "grad_norm": 0.5030350089073181, + "learning_rate": 0.00026668549229760065, + "loss": 0.534504508972168, + "mean_token_accuracy": 0.8385435181856156, + "num_tokens": 2204591.0, + "step": 950 + }, + { + "entropy": 0.6195934276282787, + "epoch": 2.4103802051901027, + "grad_norm": 0.5366560816764832, + "learning_rate": 0.00026401244383751063, + "loss": 0.5358476257324218, + "mean_token_accuracy": 0.8378069770336151, + "num_tokens": 2318101.0, + "step": 1000 + }, + { + "entropy": 0.6122904714941978, + "epoch": 2.5310802655401328, + "grad_norm": 0.5217883586883545, + "learning_rate": 0.0002611206166347852, + "loss": 0.5301174545288085, + "mean_token_accuracy": 0.8405694562196732, + "num_tokens": 2434784.0, + "step": 1050 + }, + { + "entropy": 0.6195291511714458, + "epoch": 2.651780325890163, + "grad_norm": 0.4354388117790222, + "learning_rate": 0.00025801512476177085, + "loss": 0.5370507431030274, + "mean_token_accuracy": 0.8366325795650482, + "num_tokens": 2551589.0, + "step": 1100 + }, + { + "entropy": 0.6238582208752632, + "epoch": 2.772480386240193, + "grad_norm": 0.4072394371032715, + "learning_rate": 0.00025470146014761687, + "loss": 0.5405396270751953, + "mean_token_accuracy": 0.8382216826081276, + "num_tokens": 2663427.0, + "step": 1150 + }, + { + "entropy": 0.6227807649970054, + "epoch": 2.8931804465902236, + "grad_norm": 0.4396214187145233, + "learning_rate": 0.0002511854828660347, + "loss": 0.5333737182617188, + "mean_token_accuracy": 0.8379432842135429, + "num_tokens": 2777993.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6311895140771115, + "eval_loss": 0.6008551716804504, + "eval_mean_token_accuracy": 0.828147679567337, + "eval_num_tokens": 2881707.0, + "eval_runtime": 89.7157, + "eval_samples_per_second": 15.828, + "eval_steps_per_second": 1.984, + "step": 1245 + } + ], + "logging_steps": 50, + "max_steps": 4150, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.195816276764672e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/README.md b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/adapter_config.json b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f34a9da6fde155c2a6d8fec9848a329fa629c1a0 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.04879530666124931, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "gate_proj", + "q_proj", + "k_proj", + "down_proj", + "up_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/chat_template.jinja b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/tokenizer_config.json b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/trainer_state.json b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..387fc51aeaaeb8301b7ba2f229b9916d88991f24 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1660/trainer_state.json @@ -0,0 +1,408 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1660, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.1012717413902284, + "epoch": 0.12070006035003017, + "grad_norm": 1.2028698921203613, + "learning_rate": 3.313127220682932e-05, + "loss": 1.9106788635253906, + "mean_token_accuracy": 0.6147816297411919, + "num_tokens": 117421.0, + "step": 50 + }, + { + "entropy": 1.0568797352910042, + "epoch": 0.24140012070006034, + "grad_norm": 0.707794725894928, + "learning_rate": 6.693869282604292e-05, + "loss": 0.9455691528320312, + "mean_token_accuracy": 0.7514659878611565, + "num_tokens": 235894.0, + "step": 100 + }, + { + "entropy": 0.8912339144945145, + "epoch": 0.3621001810500905, + "grad_norm": 0.6819370985031128, + "learning_rate": 0.0001007461134452565, + "loss": 0.7901513671875, + "mean_token_accuracy": 0.7814472052454948, + "num_tokens": 354448.0, + "step": 150 + }, + { + "entropy": 0.8389395973086358, + "epoch": 0.4828002414001207, + "grad_norm": 0.6772453188896179, + "learning_rate": 0.0001345535340644701, + "loss": 0.7409224700927735, + "mean_token_accuracy": 0.7919744056463242, + "num_tokens": 472123.0, + "step": 200 + }, + { + "entropy": 0.818748975098133, + "epoch": 0.6035003017501509, + "grad_norm": 0.7662261724472046, + "learning_rate": 0.0001683609546836837, + "loss": 0.7154017639160156, + "mean_token_accuracy": 0.7969402438402176, + "num_tokens": 583033.0, + "step": 250 + }, + { + "entropy": 0.7894713908433915, + "epoch": 0.724200362100181, + "grad_norm": 0.6039052605628967, + "learning_rate": 0.0002021683753028973, + "loss": 0.6953791046142578, + "mean_token_accuracy": 0.8036386013031006, + "num_tokens": 702243.0, + "step": 300 + }, + { + "entropy": 0.7873514747619629, + "epoch": 0.8449004224502112, + "grad_norm": 0.5863620042800903, + "learning_rate": 0.00023597579592211088, + "loss": 0.6996137237548828, + "mean_token_accuracy": 0.8034878405928612, + "num_tokens": 818352.0, + "step": 350 + }, + { + "entropy": 0.7764339351654053, + "epoch": 0.9656004828002414, + "grad_norm": 0.49791479110717773, + "learning_rate": 0.0002697832165413245, + "loss": 0.6787944793701172, + "mean_token_accuracy": 0.8066928315162659, + "num_tokens": 926964.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7318378298470144, + "eval_loss": 0.6790198683738708, + "eval_mean_token_accuracy": 0.8080558733323987, + "eval_num_tokens": 960569.0, + "eval_runtime": 88.4043, + "eval_samples_per_second": 16.063, + "eval_steps_per_second": 2.013, + "step": 415 + }, + { + "entropy": 0.7384049266122924, + "epoch": 1.0844900422450212, + "grad_norm": 0.5788671970367432, + "learning_rate": 0.0002805442221888677, + "loss": 0.6423238372802734, + "mean_token_accuracy": 0.8125922885643044, + "num_tokens": 1044171.0, + "step": 450 + }, + { + "entropy": 0.7178834271430969, + "epoch": 1.2051901025950513, + "grad_norm": 0.5945919752120972, + "learning_rate": 0.00028025154397167963, + "loss": 0.628162841796875, + "mean_token_accuracy": 0.8169734939932823, + "num_tokens": 1161114.0, + "step": 500 + }, + { + "entropy": 0.7209221491217613, + "epoch": 1.3258901629450814, + "grad_norm": 0.49619191884994507, + "learning_rate": 0.000279711368859939, + "loss": 0.6290118408203125, + "mean_token_accuracy": 0.8175607281923294, + "num_tokens": 1278659.0, + "step": 550 + }, + { + "entropy": 0.7152036108076573, + "epoch": 1.4465902232951118, + "grad_norm": 0.5819481015205383, + "learning_rate": 0.00027892465213015695, + "loss": 0.6237701416015625, + "mean_token_accuracy": 0.8183089861273766, + "num_tokens": 1388746.0, + "step": 600 + }, + { + "entropy": 0.706106402873993, + "epoch": 1.567290283645142, + "grad_norm": 0.5964105129241943, + "learning_rate": 0.0002778927850571194, + "loss": 0.6117628479003906, + "mean_token_accuracy": 0.8191776639223098, + "num_tokens": 1503454.0, + "step": 650 + }, + { + "entropy": 0.6981569758057594, + "epoch": 1.687990343995172, + "grad_norm": 0.4185747504234314, + "learning_rate": 0.0002766175924534771, + "loss": 0.6114554214477539, + "mean_token_accuracy": 0.8221894565224648, + "num_tokens": 1620874.0, + "step": 700 + }, + { + "entropy": 0.6924624507129192, + "epoch": 1.8086904043452021, + "grad_norm": 0.5334014296531677, + "learning_rate": 0.0002751013294426428, + "loss": 0.6072402572631836, + "mean_token_accuracy": 0.8222330984473228, + "num_tokens": 1742527.0, + "step": 750 + }, + { + "entropy": 0.6838067446649074, + "epoch": 1.9293904646952322, + "grad_norm": 0.4910935163497925, + "learning_rate": 0.0002733466774707023, + "loss": 0.5899914169311523, + "mean_token_accuracy": 0.8244012910127639, + "num_tokens": 1856624.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6870943672871321, + "eval_loss": 0.6236395835876465, + "eval_mean_token_accuracy": 0.8221416041422426, + "eval_num_tokens": 1921138.0, + "eval_runtime": 89.423, + "eval_samples_per_second": 15.88, + "eval_steps_per_second": 1.991, + "step": 830 + }, + { + "entropy": 0.6511646548503547, + "epoch": 2.048280024140012, + "grad_norm": 0.3761918246746063, + "learning_rate": 0.0002713567395643921, + "loss": 0.5689863586425781, + "mean_token_accuracy": 0.830169784538637, + "num_tokens": 1971453.0, + "step": 850 + }, + { + "entropy": 0.6192687456309796, + "epoch": 2.1689800844900424, + "grad_norm": 0.5916118025779724, + "learning_rate": 0.0002691350348435296, + "loss": 0.5307585906982422, + "mean_token_accuracy": 0.8394980162382126, + "num_tokens": 2089506.0, + "step": 900 + }, + { + "entropy": 0.6199816593527794, + "epoch": 2.2896801448400725, + "grad_norm": 0.5030350089073181, + "learning_rate": 0.00026668549229760065, + "loss": 0.534504508972168, + "mean_token_accuracy": 0.8385435181856156, + "num_tokens": 2204591.0, + "step": 950 + }, + { + "entropy": 0.6195934276282787, + "epoch": 2.4103802051901027, + "grad_norm": 0.5366560816764832, + "learning_rate": 0.00026401244383751063, + "loss": 0.5358476257324218, + "mean_token_accuracy": 0.8378069770336151, + "num_tokens": 2318101.0, + "step": 1000 + }, + { + "entropy": 0.6122904714941978, + "epoch": 2.5310802655401328, + "grad_norm": 0.5217883586883545, + "learning_rate": 0.0002611206166347852, + "loss": 0.5301174545288085, + "mean_token_accuracy": 0.8405694562196732, + "num_tokens": 2434784.0, + "step": 1050 + }, + { + "entropy": 0.6195291511714458, + "epoch": 2.651780325890163, + "grad_norm": 0.4354388117790222, + "learning_rate": 0.00025801512476177085, + "loss": 0.5370507431030274, + "mean_token_accuracy": 0.8366325795650482, + "num_tokens": 2551589.0, + "step": 1100 + }, + { + "entropy": 0.6238582208752632, + "epoch": 2.772480386240193, + "grad_norm": 0.4072394371032715, + "learning_rate": 0.00025470146014761687, + "loss": 0.5405396270751953, + "mean_token_accuracy": 0.8382216826081276, + "num_tokens": 2663427.0, + "step": 1150 + }, + { + "entropy": 0.6227807649970054, + "epoch": 2.8931804465902236, + "grad_norm": 0.4396214187145233, + "learning_rate": 0.0002511854828660347, + "loss": 0.5333737182617188, + "mean_token_accuracy": 0.8379432842135429, + "num_tokens": 2777993.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6311895140771115, + "eval_loss": 0.6008551716804504, + "eval_mean_token_accuracy": 0.828147679567337, + "eval_num_tokens": 2881707.0, + "eval_runtime": 89.7157, + "eval_samples_per_second": 15.828, + "eval_steps_per_second": 1.984, + "step": 1245 + }, + { + "entropy": 0.6138979556596824, + "epoch": 3.012070006035003, + "grad_norm": 0.42796361446380615, + "learning_rate": 0.0002474734107720086, + "loss": 0.5283173751831055, + "mean_token_accuracy": 0.8406102031015502, + "num_tokens": 2894429.0, + "step": 1250 + }, + { + "entropy": 0.5328768572211265, + "epoch": 3.1327700663850333, + "grad_norm": 0.5509935617446899, + "learning_rate": 0.00024357180850578605, + "loss": 0.4471405410766602, + "mean_token_accuracy": 0.8597650766372681, + "num_tokens": 3016693.0, + "step": 1300 + }, + { + "entropy": 0.5447257669270038, + "epoch": 3.2534701267350634, + "grad_norm": 0.5602411031723022, + "learning_rate": 0.00023948757588359315, + "loss": 0.4516656494140625, + "mean_token_accuracy": 0.8596653190255165, + "num_tokens": 3139397.0, + "step": 1350 + }, + { + "entropy": 0.5670975083112717, + "epoch": 3.3741701870850935, + "grad_norm": 0.6628665924072266, + "learning_rate": 0.00023522793569560538, + "loss": 0.4664581298828125, + "mean_token_accuracy": 0.8543756544589997, + "num_tokens": 3250957.0, + "step": 1400 + }, + { + "entropy": 0.5511669707298279, + "epoch": 3.4948702474351236, + "grad_norm": 0.5638546347618103, + "learning_rate": 0.00023080042093275341, + "loss": 0.45183868408203126, + "mean_token_accuracy": 0.8568270960450173, + "num_tokens": 3364000.0, + "step": 1450 + }, + { + "entropy": 0.5535736651718617, + "epoch": 3.6155703077851538, + "grad_norm": 0.5526077747344971, + "learning_rate": 0.00022621286146495143, + "loss": 0.46557418823242186, + "mean_token_accuracy": 0.8565032765269279, + "num_tokens": 3476752.0, + "step": 1500 + }, + { + "entropy": 0.5531842482089996, + "epoch": 3.736270368135184, + "grad_norm": 0.5669108033180237, + "learning_rate": 0.00022147337019430842, + "loss": 0.46803142547607424, + "mean_token_accuracy": 0.85505122423172, + "num_tokens": 3589765.0, + "step": 1550 + }, + { + "entropy": 0.5539099664986133, + "epoch": 3.856970428485214, + "grad_norm": 0.5953419804573059, + "learning_rate": 0.00021659032870780862, + "loss": 0.4706835174560547, + "mean_token_accuracy": 0.8534781444072723, + "num_tokens": 3704401.0, + "step": 1600 + }, + { + "entropy": 0.5504449516534805, + "epoch": 3.9776704888352445, + "grad_norm": 0.4922821819782257, + "learning_rate": 0.00021157237245483462, + "loss": 0.46299037933349607, + "mean_token_accuracy": 0.8553253355622291, + "num_tokens": 3822904.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5782078034087513, + "eval_loss": 0.6006284952163696, + "eval_mean_token_accuracy": 0.8304933661155487, + "eval_num_tokens": 3842276.0, + "eval_runtime": 89.5244, + "eval_samples_per_second": 15.862, + "eval_steps_per_second": 1.988, + "step": 1660 + } + ], + "logging_steps": 50, + "max_steps": 4150, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5954231682999296e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/README.md b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/adapter_config.json b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f34a9da6fde155c2a6d8fec9848a329fa629c1a0 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.04879530666124931, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "gate_proj", + "q_proj", + "k_proj", + "down_proj", + "up_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/chat_template.jinja b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2075/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c0b760d461b3f157686e7e7bd177dcc2ffd4f503 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1206/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1206, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.1666419465465037e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3a92ee880d12f4e2435bf456a9fedcb80a94a44e --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1608/trainer_state.json @@ -0,0 +1,398 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1608, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + }, + { + "entropy": 0.42975082195768455, + "epoch": 3.1095890410958904, + "grad_norm": 0.44258150458335876, + "learning_rate": 0.0003441496479386561, + "loss": 0.3603226852416992, + "mean_token_accuracy": 0.8864873057664043, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.425300100594759, + "epoch": 3.2341220423412205, + "grad_norm": 0.46550971269607544, + "learning_rate": 0.000338256958523054, + "loss": 0.3541151809692383, + "mean_token_accuracy": 0.8874222806096077, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.4290286618471146, + "epoch": 3.3586550435865505, + "grad_norm": 0.5829023718833923, + "learning_rate": 0.00033209920599177326, + "loss": 0.35764781951904295, + "mean_token_accuracy": 0.8852742439508439, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.4331103308498859, + "epoch": 3.4831880448318806, + "grad_norm": 0.494881272315979, + "learning_rate": 0.00032568799565199543, + "loss": 0.3636521530151367, + "mean_token_accuracy": 0.8851453733444213, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.42992967620491984, + "epoch": 3.6077210460772102, + "grad_norm": 0.47595155239105225, + "learning_rate": 0.00031903541049423063, + "loss": 0.36061588287353513, + "mean_token_accuracy": 0.8861182153224945, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.44519103825092315, + "epoch": 3.7322540473225407, + "grad_norm": 0.4148159623146057, + "learning_rate": 0.0003121539884199152, + "loss": 0.37374584197998045, + "mean_token_accuracy": 0.882867374420166, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.439686142206192, + "epoch": 3.8567870485678704, + "grad_norm": 0.3918115794658661, + "learning_rate": 0.0003050566986116549, + "loss": 0.3728714752197266, + "mean_token_accuracy": 0.8835265552997589, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.4370816157758236, + "epoch": 3.9813200498132004, + "grad_norm": 0.5257322192192078, + "learning_rate": 0.0002977569170906462, + "loss": 0.36963958740234376, + "mean_token_accuracy": 0.883085435628891, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48418363807506337, + "eval_loss": 0.5320115089416504, + "eval_mean_token_accuracy": 0.853746736812037, + "eval_num_tokens": 3760116.0, + "eval_runtime": 80.2791, + "eval_samples_per_second": 17.14, + "eval_steps_per_second": 2.143, + "step": 1608 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5528184168970035e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1a6cff6288c16ba99f551c54b5bc43fa34969502 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2010/trainer_state.json @@ -0,0 +1,489 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2010, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + }, + { + "entropy": 0.42975082195768455, + "epoch": 3.1095890410958904, + "grad_norm": 0.44258150458335876, + "learning_rate": 0.0003441496479386561, + "loss": 0.3603226852416992, + "mean_token_accuracy": 0.8864873057664043, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.425300100594759, + "epoch": 3.2341220423412205, + "grad_norm": 0.46550971269607544, + "learning_rate": 0.000338256958523054, + "loss": 0.3541151809692383, + "mean_token_accuracy": 0.8874222806096077, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.4290286618471146, + "epoch": 3.3586550435865505, + "grad_norm": 0.5829023718833923, + "learning_rate": 0.00033209920599177326, + "loss": 0.35764781951904295, + "mean_token_accuracy": 0.8852742439508439, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.4331103308498859, + "epoch": 3.4831880448318806, + "grad_norm": 0.494881272315979, + "learning_rate": 0.00032568799565199543, + "loss": 0.3636521530151367, + "mean_token_accuracy": 0.8851453733444213, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.42992967620491984, + "epoch": 3.6077210460772102, + "grad_norm": 0.47595155239105225, + "learning_rate": 0.00031903541049423063, + "loss": 0.36061588287353513, + "mean_token_accuracy": 0.8861182153224945, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.44519103825092315, + "epoch": 3.7322540473225407, + "grad_norm": 0.4148159623146057, + "learning_rate": 0.0003121539884199152, + "loss": 0.37374584197998045, + "mean_token_accuracy": 0.882867374420166, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.439686142206192, + "epoch": 3.8567870485678704, + "grad_norm": 0.3918115794658661, + "learning_rate": 0.0003050566986116549, + "loss": 0.3728714752197266, + "mean_token_accuracy": 0.8835265552997589, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.4370816157758236, + "epoch": 3.9813200498132004, + "grad_norm": 0.5257322192192078, + "learning_rate": 0.0002977569170906462, + "loss": 0.36963958740234376, + "mean_token_accuracy": 0.883085435628891, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48418363807506337, + "eval_loss": 0.5320115089416504, + "eval_mean_token_accuracy": 0.853746736812037, + "eval_num_tokens": 3760116.0, + "eval_runtime": 80.2791, + "eval_samples_per_second": 17.14, + "eval_steps_per_second": 2.143, + "step": 1608 + }, + { + "entropy": 0.35559599267111885, + "epoch": 4.104607721046078, + "grad_norm": 0.5824428796768188, + "learning_rate": 0.0002902684015073434, + "loss": 0.2808858108520508, + "mean_token_accuracy": 0.9084500258017068, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.34071226201951504, + "epoch": 4.229140722291407, + "grad_norm": 0.4689781069755554, + "learning_rate": 0.0002826052652128818, + "loss": 0.2676906204223633, + "mean_token_accuracy": 0.9113168820738793, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.339734458848834, + "epoch": 4.353673723536737, + "grad_norm": 0.5321401953697205, + "learning_rate": 0.0002747819506601238, + "loss": 0.26818386077880857, + "mean_token_accuracy": 0.9111532375216485, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3525573747605085, + "epoch": 4.478206724782067, + "grad_norm": 0.5021588206291199, + "learning_rate": 0.0002668132021844582, + "loss": 0.27898569107055665, + "mean_token_accuracy": 0.9079484361410141, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.35562075704336166, + "epoch": 4.602739726027397, + "grad_norm": 0.47950589656829834, + "learning_rate": 0.000258714038215652, + "loss": 0.28017072677612304, + "mean_token_accuracy": 0.9064878270030021, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.3466539917141199, + "epoch": 4.7272727272727275, + "grad_norm": 0.4677203297615051, + "learning_rate": 0.00025049972297312516, + "loss": 0.27576370239257814, + "mean_token_accuracy": 0.9086684882640839, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.34570303693413734, + "epoch": 4.851805728518057, + "grad_norm": 0.4588606357574463, + "learning_rate": 0.0002421857376979947, + "loss": 0.2746767234802246, + "mean_token_accuracy": 0.9086508122086525, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3503329714387655, + "epoch": 4.976338729763388, + "grad_norm": 0.5665764212608337, + "learning_rate": 0.00023378775147610495, + "loss": 0.27303371429443357, + "mean_token_accuracy": 0.9096201673150063, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.41260595723640087, + "eval_loss": 0.5472355484962463, + "eval_mean_token_accuracy": 0.8565515792647074, + "eval_num_tokens": 4700145.0, + "eval_runtime": 80.2872, + "eval_samples_per_second": 17.138, + "eval_steps_per_second": 2.142, + "step": 2010 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9406889736070554e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f04791c89b9ad1be3e5ba0876552aafbdeb6114a --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2412/trainer_state.json @@ -0,0 +1,580 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2412, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + }, + { + "entropy": 0.42975082195768455, + "epoch": 3.1095890410958904, + "grad_norm": 0.44258150458335876, + "learning_rate": 0.0003441496479386561, + "loss": 0.3603226852416992, + "mean_token_accuracy": 0.8864873057664043, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.425300100594759, + "epoch": 3.2341220423412205, + "grad_norm": 0.46550971269607544, + "learning_rate": 0.000338256958523054, + "loss": 0.3541151809692383, + "mean_token_accuracy": 0.8874222806096077, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.4290286618471146, + "epoch": 3.3586550435865505, + "grad_norm": 0.5829023718833923, + "learning_rate": 0.00033209920599177326, + "loss": 0.35764781951904295, + "mean_token_accuracy": 0.8852742439508439, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.4331103308498859, + "epoch": 3.4831880448318806, + "grad_norm": 0.494881272315979, + "learning_rate": 0.00032568799565199543, + "loss": 0.3636521530151367, + "mean_token_accuracy": 0.8851453733444213, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.42992967620491984, + "epoch": 3.6077210460772102, + "grad_norm": 0.47595155239105225, + "learning_rate": 0.00031903541049423063, + "loss": 0.36061588287353513, + "mean_token_accuracy": 0.8861182153224945, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.44519103825092315, + "epoch": 3.7322540473225407, + "grad_norm": 0.4148159623146057, + "learning_rate": 0.0003121539884199152, + "loss": 0.37374584197998045, + "mean_token_accuracy": 0.882867374420166, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.439686142206192, + "epoch": 3.8567870485678704, + "grad_norm": 0.3918115794658661, + "learning_rate": 0.0003050566986116549, + "loss": 0.3728714752197266, + "mean_token_accuracy": 0.8835265552997589, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.4370816157758236, + "epoch": 3.9813200498132004, + "grad_norm": 0.5257322192192078, + "learning_rate": 0.0002977569170906462, + "loss": 0.36963958740234376, + "mean_token_accuracy": 0.883085435628891, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48418363807506337, + "eval_loss": 0.5320115089416504, + "eval_mean_token_accuracy": 0.853746736812037, + "eval_num_tokens": 3760116.0, + "eval_runtime": 80.2791, + "eval_samples_per_second": 17.14, + "eval_steps_per_second": 2.143, + "step": 1608 + }, + { + "entropy": 0.35559599267111885, + "epoch": 4.104607721046078, + "grad_norm": 0.5824428796768188, + "learning_rate": 0.0002902684015073434, + "loss": 0.2808858108520508, + "mean_token_accuracy": 0.9084500258017068, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.34071226201951504, + "epoch": 4.229140722291407, + "grad_norm": 0.4689781069755554, + "learning_rate": 0.0002826052652128818, + "loss": 0.2676906204223633, + "mean_token_accuracy": 0.9113168820738793, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.339734458848834, + "epoch": 4.353673723536737, + "grad_norm": 0.5321401953697205, + "learning_rate": 0.0002747819506601238, + "loss": 0.26818386077880857, + "mean_token_accuracy": 0.9111532375216485, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3525573747605085, + "epoch": 4.478206724782067, + "grad_norm": 0.5021588206291199, + "learning_rate": 0.0002668132021844582, + "loss": 0.27898569107055665, + "mean_token_accuracy": 0.9079484361410141, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.35562075704336166, + "epoch": 4.602739726027397, + "grad_norm": 0.47950589656829834, + "learning_rate": 0.000258714038215652, + "loss": 0.28017072677612304, + "mean_token_accuracy": 0.9064878270030021, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.3466539917141199, + "epoch": 4.7272727272727275, + "grad_norm": 0.4677203297615051, + "learning_rate": 0.00025049972297312516, + "loss": 0.27576370239257814, + "mean_token_accuracy": 0.9086684882640839, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.34570303693413734, + "epoch": 4.851805728518057, + "grad_norm": 0.4588606357574463, + "learning_rate": 0.0002421857376979947, + "loss": 0.2746767234802246, + "mean_token_accuracy": 0.9086508122086525, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3503329714387655, + "epoch": 4.976338729763388, + "grad_norm": 0.5665764212608337, + "learning_rate": 0.00023378775147610495, + "loss": 0.27303371429443357, + "mean_token_accuracy": 0.9096201673150063, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.41260595723640087, + "eval_loss": 0.5472355484962463, + "eval_mean_token_accuracy": 0.8565515792647074, + "eval_num_tokens": 4700145.0, + "eval_runtime": 80.2872, + "eval_samples_per_second": 17.138, + "eval_steps_per_second": 2.142, + "step": 2010 + }, + { + "entropy": 0.2732633537415302, + "epoch": 5.099626400996264, + "grad_norm": 0.567966639995575, + "learning_rate": 0.0002253215917070334, + "loss": 0.19740861892700196, + "mean_token_accuracy": 0.9340365438750295, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.24825763009488583, + "epoch": 5.224159402241594, + "grad_norm": 0.6070764064788818, + "learning_rate": 0.00021680321427472769, + "loss": 0.17449329376220704, + "mean_token_accuracy": 0.9399480578303337, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.26008583076298236, + "epoch": 5.348692403486924, + "grad_norm": 0.4653824269771576, + "learning_rate": 0.00020824867347599314, + "loss": 0.1837337875366211, + "mean_token_accuracy": 0.9369171789288521, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.26238673031330106, + "epoch": 5.473225404732254, + "grad_norm": 0.5280150771141052, + "learning_rate": 0.0001996740917635043, + "loss": 0.18675100326538085, + "mean_token_accuracy": 0.936633392572403, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.26483887039124965, + "epoch": 5.597758405977584, + "grad_norm": 0.4668729305267334, + "learning_rate": 0.00019109562936036546, + "loss": 0.18501157760620118, + "mean_token_accuracy": 0.9365152379870415, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.2524403877556324, + "epoch": 5.722291407222914, + "grad_norm": 0.5033115148544312, + "learning_rate": 0.00018252945380348672, + "loss": 0.1812763023376465, + "mean_token_accuracy": 0.9381375980377197, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.25848230116069315, + "epoch": 5.846824408468244, + "grad_norm": 0.6071104407310486, + "learning_rate": 0.0001739917094731765, + "loss": 0.18280351638793946, + "mean_token_accuracy": 0.9371961706876755, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.2512618475407362, + "epoch": 5.971357409713574, + "grad_norm": 0.5169245004653931, + "learning_rate": 0.00016549848716637528, + "loss": 0.1812678337097168, + "mean_token_accuracy": 0.93798923432827, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35272262487993683, + "eval_loss": 0.6165693998336792, + "eval_mean_token_accuracy": 0.8541817599257757, + "eval_num_tokens": 5640174.0, + "eval_runtime": 80.0797, + "eval_samples_per_second": 17.183, + "eval_steps_per_second": 2.148, + "step": 2412 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.3282680816735027e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..612e893d7b070888f9b8cfe654dd300a290a733f --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2814/trainer_state.json @@ -0,0 +1,671 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2814, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + }, + { + "entropy": 0.42975082195768455, + "epoch": 3.1095890410958904, + "grad_norm": 0.44258150458335876, + "learning_rate": 0.0003441496479386561, + "loss": 0.3603226852416992, + "mean_token_accuracy": 0.8864873057664043, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.425300100594759, + "epoch": 3.2341220423412205, + "grad_norm": 0.46550971269607544, + "learning_rate": 0.000338256958523054, + "loss": 0.3541151809692383, + "mean_token_accuracy": 0.8874222806096077, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.4290286618471146, + "epoch": 3.3586550435865505, + "grad_norm": 0.5829023718833923, + "learning_rate": 0.00033209920599177326, + "loss": 0.35764781951904295, + "mean_token_accuracy": 0.8852742439508439, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.4331103308498859, + "epoch": 3.4831880448318806, + "grad_norm": 0.494881272315979, + "learning_rate": 0.00032568799565199543, + "loss": 0.3636521530151367, + "mean_token_accuracy": 0.8851453733444213, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.42992967620491984, + "epoch": 3.6077210460772102, + "grad_norm": 0.47595155239105225, + "learning_rate": 0.00031903541049423063, + "loss": 0.36061588287353513, + "mean_token_accuracy": 0.8861182153224945, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.44519103825092315, + "epoch": 3.7322540473225407, + "grad_norm": 0.4148159623146057, + "learning_rate": 0.0003121539884199152, + "loss": 0.37374584197998045, + "mean_token_accuracy": 0.882867374420166, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.439686142206192, + "epoch": 3.8567870485678704, + "grad_norm": 0.3918115794658661, + "learning_rate": 0.0003050566986116549, + "loss": 0.3728714752197266, + "mean_token_accuracy": 0.8835265552997589, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.4370816157758236, + "epoch": 3.9813200498132004, + "grad_norm": 0.5257322192192078, + "learning_rate": 0.0002977569170906462, + "loss": 0.36963958740234376, + "mean_token_accuracy": 0.883085435628891, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48418363807506337, + "eval_loss": 0.5320115089416504, + "eval_mean_token_accuracy": 0.853746736812037, + "eval_num_tokens": 3760116.0, + "eval_runtime": 80.2791, + "eval_samples_per_second": 17.14, + "eval_steps_per_second": 2.143, + "step": 1608 + }, + { + "entropy": 0.35559599267111885, + "epoch": 4.104607721046078, + "grad_norm": 0.5824428796768188, + "learning_rate": 0.0002902684015073434, + "loss": 0.2808858108520508, + "mean_token_accuracy": 0.9084500258017068, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.34071226201951504, + "epoch": 4.229140722291407, + "grad_norm": 0.4689781069755554, + "learning_rate": 0.0002826052652128818, + "loss": 0.2676906204223633, + "mean_token_accuracy": 0.9113168820738793, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.339734458848834, + "epoch": 4.353673723536737, + "grad_norm": 0.5321401953697205, + "learning_rate": 0.0002747819506601238, + "loss": 0.26818386077880857, + "mean_token_accuracy": 0.9111532375216485, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3525573747605085, + "epoch": 4.478206724782067, + "grad_norm": 0.5021588206291199, + "learning_rate": 0.0002668132021844582, + "loss": 0.27898569107055665, + "mean_token_accuracy": 0.9079484361410141, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.35562075704336166, + "epoch": 4.602739726027397, + "grad_norm": 0.47950589656829834, + "learning_rate": 0.000258714038215652, + "loss": 0.28017072677612304, + "mean_token_accuracy": 0.9064878270030021, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.3466539917141199, + "epoch": 4.7272727272727275, + "grad_norm": 0.4677203297615051, + "learning_rate": 0.00025049972297312516, + "loss": 0.27576370239257814, + "mean_token_accuracy": 0.9086684882640839, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.34570303693413734, + "epoch": 4.851805728518057, + "grad_norm": 0.4588606357574463, + "learning_rate": 0.0002421857376979947, + "loss": 0.2746767234802246, + "mean_token_accuracy": 0.9086508122086525, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3503329714387655, + "epoch": 4.976338729763388, + "grad_norm": 0.5665764212608337, + "learning_rate": 0.00023378775147610495, + "loss": 0.27303371429443357, + "mean_token_accuracy": 0.9096201673150063, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.41260595723640087, + "eval_loss": 0.5472355484962463, + "eval_mean_token_accuracy": 0.8565515792647074, + "eval_num_tokens": 4700145.0, + "eval_runtime": 80.2872, + "eval_samples_per_second": 17.138, + "eval_steps_per_second": 2.142, + "step": 2010 + }, + { + "entropy": 0.2732633537415302, + "epoch": 5.099626400996264, + "grad_norm": 0.567966639995575, + "learning_rate": 0.0002253215917070334, + "loss": 0.19740861892700196, + "mean_token_accuracy": 0.9340365438750295, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.24825763009488583, + "epoch": 5.224159402241594, + "grad_norm": 0.6070764064788818, + "learning_rate": 0.00021680321427472769, + "loss": 0.17449329376220704, + "mean_token_accuracy": 0.9399480578303337, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.26008583076298236, + "epoch": 5.348692403486924, + "grad_norm": 0.4653824269771576, + "learning_rate": 0.00020824867347599314, + "loss": 0.1837337875366211, + "mean_token_accuracy": 0.9369171789288521, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.26238673031330106, + "epoch": 5.473225404732254, + "grad_norm": 0.5280150771141052, + "learning_rate": 0.0001996740917635043, + "loss": 0.18675100326538085, + "mean_token_accuracy": 0.936633392572403, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.26483887039124965, + "epoch": 5.597758405977584, + "grad_norm": 0.4668729305267334, + "learning_rate": 0.00019109562936036546, + "loss": 0.18501157760620118, + "mean_token_accuracy": 0.9365152379870415, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.2524403877556324, + "epoch": 5.722291407222914, + "grad_norm": 0.5033115148544312, + "learning_rate": 0.00018252945380348672, + "loss": 0.1812763023376465, + "mean_token_accuracy": 0.9381375980377197, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.25848230116069315, + "epoch": 5.846824408468244, + "grad_norm": 0.6071104407310486, + "learning_rate": 0.0001739917094731765, + "loss": 0.18280351638793946, + "mean_token_accuracy": 0.9371961706876755, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.2512618475407362, + "epoch": 5.971357409713574, + "grad_norm": 0.5169245004653931, + "learning_rate": 0.00016549848716637528, + "loss": 0.1812678337097168, + "mean_token_accuracy": 0.93798923432827, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35272262487993683, + "eval_loss": 0.6165693998336792, + "eval_mean_token_accuracy": 0.8541817599257757, + "eval_num_tokens": 5640174.0, + "eval_runtime": 80.0797, + "eval_samples_per_second": 17.183, + "eval_steps_per_second": 2.148, + "step": 2412 + }, + { + "entropy": 0.20745287752813762, + "epoch": 6.094645080946451, + "grad_norm": 0.33874237537384033, + "learning_rate": 0.000157065793770877, + "loss": 0.1288090705871582, + "mean_token_accuracy": 0.9561566412448883, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.18590272210538386, + "epoch": 6.219178082191781, + "grad_norm": 0.43293333053588867, + "learning_rate": 0.0001487095220976913, + "loss": 0.11404885292053223, + "mean_token_accuracy": 0.961484112739563, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.18391913663595916, + "epoch": 6.34371108343711, + "grad_norm": 0.38351935148239136, + "learning_rate": 0.00014044542092840146, + "loss": 0.11131843566894531, + "mean_token_accuracy": 0.9621073800325394, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.17878302346915007, + "epoch": 6.468244084682441, + "grad_norm": 0.430851012468338, + "learning_rate": 0.00013228906533397082, + "loss": 0.10821202278137207, + "mean_token_accuracy": 0.9637334972620011, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.18323683477938174, + "epoch": 6.592777085927771, + "grad_norm": 0.3986610174179077, + "learning_rate": 0.00012425582732093497, + "loss": 0.11400936126708984, + "mean_token_accuracy": 0.9619395446777343, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.18330949898809196, + "epoch": 6.717310087173101, + "grad_norm": 0.5595681667327881, + "learning_rate": 0.0001163608468603028, + "loss": 0.113320951461792, + "mean_token_accuracy": 0.9610482943058014, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.18616238351911307, + "epoch": 6.841843088418431, + "grad_norm": 0.45847031474113464, + "learning_rate": 0.00010861900335376734, + "loss": 0.11270190238952636, + "mean_token_accuracy": 0.9607535809278488, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.17851211048662663, + "epoch": 6.966376089663761, + "grad_norm": 0.3785495162010193, + "learning_rate": 0.00010104488759100174, + "loss": 0.1098051643371582, + "mean_token_accuracy": 0.9622898560762405, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29157780076182166, + "eval_loss": 0.7041030526161194, + "eval_mean_token_accuracy": 0.8533015802156093, + "eval_num_tokens": 6580203.0, + "eval_runtime": 80.3186, + "eval_samples_per_second": 17.132, + "eval_steps_per_second": 2.141, + "step": 2814 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.713509679512576e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d5aa0e6f2f73336eecbaaaf2dddc342931503406 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3216/trainer_state.json @@ -0,0 +1,762 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3216, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + }, + { + "entropy": 0.42975082195768455, + "epoch": 3.1095890410958904, + "grad_norm": 0.44258150458335876, + "learning_rate": 0.0003441496479386561, + "loss": 0.3603226852416992, + "mean_token_accuracy": 0.8864873057664043, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.425300100594759, + "epoch": 3.2341220423412205, + "grad_norm": 0.46550971269607544, + "learning_rate": 0.000338256958523054, + "loss": 0.3541151809692383, + "mean_token_accuracy": 0.8874222806096077, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.4290286618471146, + "epoch": 3.3586550435865505, + "grad_norm": 0.5829023718833923, + "learning_rate": 0.00033209920599177326, + "loss": 0.35764781951904295, + "mean_token_accuracy": 0.8852742439508439, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.4331103308498859, + "epoch": 3.4831880448318806, + "grad_norm": 0.494881272315979, + "learning_rate": 0.00032568799565199543, + "loss": 0.3636521530151367, + "mean_token_accuracy": 0.8851453733444213, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.42992967620491984, + "epoch": 3.6077210460772102, + "grad_norm": 0.47595155239105225, + "learning_rate": 0.00031903541049423063, + "loss": 0.36061588287353513, + "mean_token_accuracy": 0.8861182153224945, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.44519103825092315, + "epoch": 3.7322540473225407, + "grad_norm": 0.4148159623146057, + "learning_rate": 0.0003121539884199152, + "loss": 0.37374584197998045, + "mean_token_accuracy": 0.882867374420166, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.439686142206192, + "epoch": 3.8567870485678704, + "grad_norm": 0.3918115794658661, + "learning_rate": 0.0003050566986116549, + "loss": 0.3728714752197266, + "mean_token_accuracy": 0.8835265552997589, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.4370816157758236, + "epoch": 3.9813200498132004, + "grad_norm": 0.5257322192192078, + "learning_rate": 0.0002977569170906462, + "loss": 0.36963958740234376, + "mean_token_accuracy": 0.883085435628891, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48418363807506337, + "eval_loss": 0.5320115089416504, + "eval_mean_token_accuracy": 0.853746736812037, + "eval_num_tokens": 3760116.0, + "eval_runtime": 80.2791, + "eval_samples_per_second": 17.14, + "eval_steps_per_second": 2.143, + "step": 1608 + }, + { + "entropy": 0.35559599267111885, + "epoch": 4.104607721046078, + "grad_norm": 0.5824428796768188, + "learning_rate": 0.0002902684015073434, + "loss": 0.2808858108520508, + "mean_token_accuracy": 0.9084500258017068, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.34071226201951504, + "epoch": 4.229140722291407, + "grad_norm": 0.4689781069755554, + "learning_rate": 0.0002826052652128818, + "loss": 0.2676906204223633, + "mean_token_accuracy": 0.9113168820738793, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.339734458848834, + "epoch": 4.353673723536737, + "grad_norm": 0.5321401953697205, + "learning_rate": 0.0002747819506601238, + "loss": 0.26818386077880857, + "mean_token_accuracy": 0.9111532375216485, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3525573747605085, + "epoch": 4.478206724782067, + "grad_norm": 0.5021588206291199, + "learning_rate": 0.0002668132021844582, + "loss": 0.27898569107055665, + "mean_token_accuracy": 0.9079484361410141, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.35562075704336166, + "epoch": 4.602739726027397, + "grad_norm": 0.47950589656829834, + "learning_rate": 0.000258714038215652, + "loss": 0.28017072677612304, + "mean_token_accuracy": 0.9064878270030021, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.3466539917141199, + "epoch": 4.7272727272727275, + "grad_norm": 0.4677203297615051, + "learning_rate": 0.00025049972297312516, + "loss": 0.27576370239257814, + "mean_token_accuracy": 0.9086684882640839, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.34570303693413734, + "epoch": 4.851805728518057, + "grad_norm": 0.4588606357574463, + "learning_rate": 0.0002421857376979947, + "loss": 0.2746767234802246, + "mean_token_accuracy": 0.9086508122086525, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3503329714387655, + "epoch": 4.976338729763388, + "grad_norm": 0.5665764212608337, + "learning_rate": 0.00023378775147610495, + "loss": 0.27303371429443357, + "mean_token_accuracy": 0.9096201673150063, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.41260595723640087, + "eval_loss": 0.5472355484962463, + "eval_mean_token_accuracy": 0.8565515792647074, + "eval_num_tokens": 4700145.0, + "eval_runtime": 80.2872, + "eval_samples_per_second": 17.138, + "eval_steps_per_second": 2.142, + "step": 2010 + }, + { + "entropy": 0.2732633537415302, + "epoch": 5.099626400996264, + "grad_norm": 0.567966639995575, + "learning_rate": 0.0002253215917070334, + "loss": 0.19740861892700196, + "mean_token_accuracy": 0.9340365438750295, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.24825763009488583, + "epoch": 5.224159402241594, + "grad_norm": 0.6070764064788818, + "learning_rate": 0.00021680321427472769, + "loss": 0.17449329376220704, + "mean_token_accuracy": 0.9399480578303337, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.26008583076298236, + "epoch": 5.348692403486924, + "grad_norm": 0.4653824269771576, + "learning_rate": 0.00020824867347599314, + "loss": 0.1837337875366211, + "mean_token_accuracy": 0.9369171789288521, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.26238673031330106, + "epoch": 5.473225404732254, + "grad_norm": 0.5280150771141052, + "learning_rate": 0.0001996740917635043, + "loss": 0.18675100326538085, + "mean_token_accuracy": 0.936633392572403, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.26483887039124965, + "epoch": 5.597758405977584, + "grad_norm": 0.4668729305267334, + "learning_rate": 0.00019109562936036546, + "loss": 0.18501157760620118, + "mean_token_accuracy": 0.9365152379870415, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.2524403877556324, + "epoch": 5.722291407222914, + "grad_norm": 0.5033115148544312, + "learning_rate": 0.00018252945380348672, + "loss": 0.1812763023376465, + "mean_token_accuracy": 0.9381375980377197, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.25848230116069315, + "epoch": 5.846824408468244, + "grad_norm": 0.6071104407310486, + "learning_rate": 0.0001739917094731765, + "loss": 0.18280351638793946, + "mean_token_accuracy": 0.9371961706876755, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.2512618475407362, + "epoch": 5.971357409713574, + "grad_norm": 0.5169245004653931, + "learning_rate": 0.00016549848716637528, + "loss": 0.1812678337097168, + "mean_token_accuracy": 0.93798923432827, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35272262487993683, + "eval_loss": 0.6165693998336792, + "eval_mean_token_accuracy": 0.8541817599257757, + "eval_num_tokens": 5640174.0, + "eval_runtime": 80.0797, + "eval_samples_per_second": 17.183, + "eval_steps_per_second": 2.148, + "step": 2412 + }, + { + "entropy": 0.20745287752813762, + "epoch": 6.094645080946451, + "grad_norm": 0.33874237537384033, + "learning_rate": 0.000157065793770877, + "loss": 0.1288090705871582, + "mean_token_accuracy": 0.9561566412448883, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.18590272210538386, + "epoch": 6.219178082191781, + "grad_norm": 0.43293333053588867, + "learning_rate": 0.0001487095220976913, + "loss": 0.11404885292053223, + "mean_token_accuracy": 0.961484112739563, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.18391913663595916, + "epoch": 6.34371108343711, + "grad_norm": 0.38351935148239136, + "learning_rate": 0.00014044542092840146, + "loss": 0.11131843566894531, + "mean_token_accuracy": 0.9621073800325394, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.17878302346915007, + "epoch": 6.468244084682441, + "grad_norm": 0.430851012468338, + "learning_rate": 0.00013228906533397082, + "loss": 0.10821202278137207, + "mean_token_accuracy": 0.9637334972620011, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.18323683477938174, + "epoch": 6.592777085927771, + "grad_norm": 0.3986610174179077, + "learning_rate": 0.00012425582732093497, + "loss": 0.11400936126708984, + "mean_token_accuracy": 0.9619395446777343, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.18330949898809196, + "epoch": 6.717310087173101, + "grad_norm": 0.5595681667327881, + "learning_rate": 0.0001163608468603028, + "loss": 0.113320951461792, + "mean_token_accuracy": 0.9610482943058014, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.18616238351911307, + "epoch": 6.841843088418431, + "grad_norm": 0.45847031474113464, + "learning_rate": 0.00010861900335376734, + "loss": 0.11270190238952636, + "mean_token_accuracy": 0.9607535809278488, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.17851211048662663, + "epoch": 6.966376089663761, + "grad_norm": 0.3785495162010193, + "learning_rate": 0.00010104488759100174, + "loss": 0.1098051643371582, + "mean_token_accuracy": 0.9622898560762405, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29157780076182166, + "eval_loss": 0.7041030526161194, + "eval_mean_token_accuracy": 0.8533015802156093, + "eval_num_tokens": 6580203.0, + "eval_runtime": 80.3186, + "eval_samples_per_second": 17.132, + "eval_steps_per_second": 2.141, + "step": 2814 + }, + { + "entropy": 0.16289814633100924, + "epoch": 7.089663760896638, + "grad_norm": 0.21091541647911072, + "learning_rate": 9.365277425089424e-05, + "loss": 0.08835570335388183, + "mean_token_accuracy": 0.9695430569576494, + "num_tokens": 6661238.0, + "step": 2850 + }, + { + "entropy": 0.1420974024385214, + "epoch": 7.214196762141968, + "grad_norm": 0.365073561668396, + "learning_rate": 8.645659499854436e-05, + "loss": 0.0793186902999878, + "mean_token_accuracy": 0.9731000065803528, + "num_tokens": 6774391.0, + "step": 2900 + }, + { + "entropy": 0.14309360027313234, + "epoch": 7.338729763387297, + "grad_norm": 0.388925164937973, + "learning_rate": 7.946991222872743e-05, + "loss": 0.07888932704925537, + "mean_token_accuracy": 0.9733487150073051, + "num_tokens": 6892967.0, + "step": 2950 + }, + { + "entropy": 0.14983555225655437, + "epoch": 7.463262764632628, + "grad_norm": 0.32667651772499084, + "learning_rate": 7.270589350530889e-05, + "loss": 0.07984059333801269, + "mean_token_accuracy": 0.9718177166581153, + "num_tokens": 7006692.0, + "step": 3000 + }, + { + "entropy": 0.14554835664108395, + "epoch": 7.587795765877957, + "grad_norm": 0.25567975640296936, + "learning_rate": 6.617728674478406e-05, + "loss": 0.0789554786682129, + "mean_token_accuracy": 0.9735771152377128, + "num_tokens": 7124768.0, + "step": 3050 + }, + { + "entropy": 0.1400925239548087, + "epoch": 7.712328767123288, + "grad_norm": 0.38630175590515137, + "learning_rate": 5.989639619071232e-05, + "loss": 0.078687162399292, + "mean_token_accuracy": 0.9736493360996247, + "num_tokens": 7242764.0, + "step": 3100 + }, + { + "entropy": 0.13647916626185178, + "epoch": 7.8368617683686175, + "grad_norm": 0.2967790961265564, + "learning_rate": 5.3875059224327037e-05, + "loss": 0.07509267330169678, + "mean_token_accuracy": 0.9741105973720551, + "num_tokens": 7366604.0, + "step": 3150 + }, + { + "entropy": 0.14620193544775248, + "epoch": 7.961394769613948, + "grad_norm": 0.2764667272567749, + "learning_rate": 4.812462405502509e-05, + "loss": 0.07793283462524414, + "mean_token_accuracy": 0.9722180238366127, + "num_tokens": 7481687.0, + "step": 3200 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2592288721785989, + "eval_loss": 0.7876929640769958, + "eval_mean_token_accuracy": 0.8539637528186621, + "eval_num_tokens": 7520232.0, + "eval_runtime": 80.8149, + "eval_samples_per_second": 17.027, + "eval_steps_per_second": 2.128, + "step": 3216 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.103242744371712e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e070d6390426aea66bd08e2ff7c747864c573f0f --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3618/trainer_state.json @@ -0,0 +1,853 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3618, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + }, + { + "entropy": 0.42975082195768455, + "epoch": 3.1095890410958904, + "grad_norm": 0.44258150458335876, + "learning_rate": 0.0003441496479386561, + "loss": 0.3603226852416992, + "mean_token_accuracy": 0.8864873057664043, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.425300100594759, + "epoch": 3.2341220423412205, + "grad_norm": 0.46550971269607544, + "learning_rate": 0.000338256958523054, + "loss": 0.3541151809692383, + "mean_token_accuracy": 0.8874222806096077, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.4290286618471146, + "epoch": 3.3586550435865505, + "grad_norm": 0.5829023718833923, + "learning_rate": 0.00033209920599177326, + "loss": 0.35764781951904295, + "mean_token_accuracy": 0.8852742439508439, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.4331103308498859, + "epoch": 3.4831880448318806, + "grad_norm": 0.494881272315979, + "learning_rate": 0.00032568799565199543, + "loss": 0.3636521530151367, + "mean_token_accuracy": 0.8851453733444213, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.42992967620491984, + "epoch": 3.6077210460772102, + "grad_norm": 0.47595155239105225, + "learning_rate": 0.00031903541049423063, + "loss": 0.36061588287353513, + "mean_token_accuracy": 0.8861182153224945, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.44519103825092315, + "epoch": 3.7322540473225407, + "grad_norm": 0.4148159623146057, + "learning_rate": 0.0003121539884199152, + "loss": 0.37374584197998045, + "mean_token_accuracy": 0.882867374420166, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.439686142206192, + "epoch": 3.8567870485678704, + "grad_norm": 0.3918115794658661, + "learning_rate": 0.0003050566986116549, + "loss": 0.3728714752197266, + "mean_token_accuracy": 0.8835265552997589, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.4370816157758236, + "epoch": 3.9813200498132004, + "grad_norm": 0.5257322192192078, + "learning_rate": 0.0002977569170906462, + "loss": 0.36963958740234376, + "mean_token_accuracy": 0.883085435628891, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48418363807506337, + "eval_loss": 0.5320115089416504, + "eval_mean_token_accuracy": 0.853746736812037, + "eval_num_tokens": 3760116.0, + "eval_runtime": 80.2791, + "eval_samples_per_second": 17.14, + "eval_steps_per_second": 2.143, + "step": 1608 + }, + { + "entropy": 0.35559599267111885, + "epoch": 4.104607721046078, + "grad_norm": 0.5824428796768188, + "learning_rate": 0.0002902684015073434, + "loss": 0.2808858108520508, + "mean_token_accuracy": 0.9084500258017068, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.34071226201951504, + "epoch": 4.229140722291407, + "grad_norm": 0.4689781069755554, + "learning_rate": 0.0002826052652128818, + "loss": 0.2676906204223633, + "mean_token_accuracy": 0.9113168820738793, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.339734458848834, + "epoch": 4.353673723536737, + "grad_norm": 0.5321401953697205, + "learning_rate": 0.0002747819506601238, + "loss": 0.26818386077880857, + "mean_token_accuracy": 0.9111532375216485, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3525573747605085, + "epoch": 4.478206724782067, + "grad_norm": 0.5021588206291199, + "learning_rate": 0.0002668132021844582, + "loss": 0.27898569107055665, + "mean_token_accuracy": 0.9079484361410141, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.35562075704336166, + "epoch": 4.602739726027397, + "grad_norm": 0.47950589656829834, + "learning_rate": 0.000258714038215652, + "loss": 0.28017072677612304, + "mean_token_accuracy": 0.9064878270030021, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.3466539917141199, + "epoch": 4.7272727272727275, + "grad_norm": 0.4677203297615051, + "learning_rate": 0.00025049972297312516, + "loss": 0.27576370239257814, + "mean_token_accuracy": 0.9086684882640839, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.34570303693413734, + "epoch": 4.851805728518057, + "grad_norm": 0.4588606357574463, + "learning_rate": 0.0002421857376979947, + "loss": 0.2746767234802246, + "mean_token_accuracy": 0.9086508122086525, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3503329714387655, + "epoch": 4.976338729763388, + "grad_norm": 0.5665764212608337, + "learning_rate": 0.00023378775147610495, + "loss": 0.27303371429443357, + "mean_token_accuracy": 0.9096201673150063, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.41260595723640087, + "eval_loss": 0.5472355484962463, + "eval_mean_token_accuracy": 0.8565515792647074, + "eval_num_tokens": 4700145.0, + "eval_runtime": 80.2872, + "eval_samples_per_second": 17.138, + "eval_steps_per_second": 2.142, + "step": 2010 + }, + { + "entropy": 0.2732633537415302, + "epoch": 5.099626400996264, + "grad_norm": 0.567966639995575, + "learning_rate": 0.0002253215917070334, + "loss": 0.19740861892700196, + "mean_token_accuracy": 0.9340365438750295, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.24825763009488583, + "epoch": 5.224159402241594, + "grad_norm": 0.6070764064788818, + "learning_rate": 0.00021680321427472769, + "loss": 0.17449329376220704, + "mean_token_accuracy": 0.9399480578303337, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.26008583076298236, + "epoch": 5.348692403486924, + "grad_norm": 0.4653824269771576, + "learning_rate": 0.00020824867347599314, + "loss": 0.1837337875366211, + "mean_token_accuracy": 0.9369171789288521, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.26238673031330106, + "epoch": 5.473225404732254, + "grad_norm": 0.5280150771141052, + "learning_rate": 0.0001996740917635043, + "loss": 0.18675100326538085, + "mean_token_accuracy": 0.936633392572403, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.26483887039124965, + "epoch": 5.597758405977584, + "grad_norm": 0.4668729305267334, + "learning_rate": 0.00019109562936036546, + "loss": 0.18501157760620118, + "mean_token_accuracy": 0.9365152379870415, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.2524403877556324, + "epoch": 5.722291407222914, + "grad_norm": 0.5033115148544312, + "learning_rate": 0.00018252945380348672, + "loss": 0.1812763023376465, + "mean_token_accuracy": 0.9381375980377197, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.25848230116069315, + "epoch": 5.846824408468244, + "grad_norm": 0.6071104407310486, + "learning_rate": 0.0001739917094731765, + "loss": 0.18280351638793946, + "mean_token_accuracy": 0.9371961706876755, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.2512618475407362, + "epoch": 5.971357409713574, + "grad_norm": 0.5169245004653931, + "learning_rate": 0.00016549848716637528, + "loss": 0.1812678337097168, + "mean_token_accuracy": 0.93798923432827, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35272262487993683, + "eval_loss": 0.6165693998336792, + "eval_mean_token_accuracy": 0.8541817599257757, + "eval_num_tokens": 5640174.0, + "eval_runtime": 80.0797, + "eval_samples_per_second": 17.183, + "eval_steps_per_second": 2.148, + "step": 2412 + }, + { + "entropy": 0.20745287752813762, + "epoch": 6.094645080946451, + "grad_norm": 0.33874237537384033, + "learning_rate": 0.000157065793770877, + "loss": 0.1288090705871582, + "mean_token_accuracy": 0.9561566412448883, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.18590272210538386, + "epoch": 6.219178082191781, + "grad_norm": 0.43293333053588867, + "learning_rate": 0.0001487095220976913, + "loss": 0.11404885292053223, + "mean_token_accuracy": 0.961484112739563, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.18391913663595916, + "epoch": 6.34371108343711, + "grad_norm": 0.38351935148239136, + "learning_rate": 0.00014044542092840146, + "loss": 0.11131843566894531, + "mean_token_accuracy": 0.9621073800325394, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.17878302346915007, + "epoch": 6.468244084682441, + "grad_norm": 0.430851012468338, + "learning_rate": 0.00013228906533397082, + "loss": 0.10821202278137207, + "mean_token_accuracy": 0.9637334972620011, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.18323683477938174, + "epoch": 6.592777085927771, + "grad_norm": 0.3986610174179077, + "learning_rate": 0.00012425582732093497, + "loss": 0.11400936126708984, + "mean_token_accuracy": 0.9619395446777343, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.18330949898809196, + "epoch": 6.717310087173101, + "grad_norm": 0.5595681667327881, + "learning_rate": 0.0001163608468603028, + "loss": 0.113320951461792, + "mean_token_accuracy": 0.9610482943058014, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.18616238351911307, + "epoch": 6.841843088418431, + "grad_norm": 0.45847031474113464, + "learning_rate": 0.00010861900335376734, + "loss": 0.11270190238952636, + "mean_token_accuracy": 0.9607535809278488, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.17851211048662663, + "epoch": 6.966376089663761, + "grad_norm": 0.3785495162010193, + "learning_rate": 0.00010104488759100174, + "loss": 0.1098051643371582, + "mean_token_accuracy": 0.9622898560762405, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29157780076182166, + "eval_loss": 0.7041030526161194, + "eval_mean_token_accuracy": 0.8533015802156093, + "eval_num_tokens": 6580203.0, + "eval_runtime": 80.3186, + "eval_samples_per_second": 17.132, + "eval_steps_per_second": 2.141, + "step": 2814 + }, + { + "entropy": 0.16289814633100924, + "epoch": 7.089663760896638, + "grad_norm": 0.21091541647911072, + "learning_rate": 9.365277425089424e-05, + "loss": 0.08835570335388183, + "mean_token_accuracy": 0.9695430569576494, + "num_tokens": 6661238.0, + "step": 2850 + }, + { + "entropy": 0.1420974024385214, + "epoch": 7.214196762141968, + "grad_norm": 0.365073561668396, + "learning_rate": 8.645659499854436e-05, + "loss": 0.0793186902999878, + "mean_token_accuracy": 0.9731000065803528, + "num_tokens": 6774391.0, + "step": 2900 + }, + { + "entropy": 0.14309360027313234, + "epoch": 7.338729763387297, + "grad_norm": 0.388925164937973, + "learning_rate": 7.946991222872743e-05, + "loss": 0.07888932704925537, + "mean_token_accuracy": 0.9733487150073051, + "num_tokens": 6892967.0, + "step": 2950 + }, + { + "entropy": 0.14983555225655437, + "epoch": 7.463262764632628, + "grad_norm": 0.32667651772499084, + "learning_rate": 7.270589350530889e-05, + "loss": 0.07984059333801269, + "mean_token_accuracy": 0.9718177166581153, + "num_tokens": 7006692.0, + "step": 3000 + }, + { + "entropy": 0.14554835664108395, + "epoch": 7.587795765877957, + "grad_norm": 0.25567975640296936, + "learning_rate": 6.617728674478406e-05, + "loss": 0.0789554786682129, + "mean_token_accuracy": 0.9735771152377128, + "num_tokens": 7124768.0, + "step": 3050 + }, + { + "entropy": 0.1400925239548087, + "epoch": 7.712328767123288, + "grad_norm": 0.38630175590515137, + "learning_rate": 5.989639619071232e-05, + "loss": 0.078687162399292, + "mean_token_accuracy": 0.9736493360996247, + "num_tokens": 7242764.0, + "step": 3100 + }, + { + "entropy": 0.13647916626185178, + "epoch": 7.8368617683686175, + "grad_norm": 0.2967790961265564, + "learning_rate": 5.3875059224327037e-05, + "loss": 0.07509267330169678, + "mean_token_accuracy": 0.9741105973720551, + "num_tokens": 7366604.0, + "step": 3150 + }, + { + "entropy": 0.14620193544775248, + "epoch": 7.961394769613948, + "grad_norm": 0.2764667272567749, + "learning_rate": 4.812462405502509e-05, + "loss": 0.07793283462524414, + "mean_token_accuracy": 0.9722180238366127, + "num_tokens": 7481687.0, + "step": 3200 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2592288721785989, + "eval_loss": 0.7876929640769958, + "eval_mean_token_accuracy": 0.8539637528186621, + "eval_num_tokens": 7520232.0, + "eval_runtime": 80.8149, + "eval_samples_per_second": 17.027, + "eval_steps_per_second": 2.128, + "step": 3216 + }, + { + "entropy": 0.1309273194220632, + "epoch": 8.084682440846825, + "grad_norm": 0.24712106585502625, + "learning_rate": 4.265592833278151e-05, + "loss": 0.06616766929626465, + "mean_token_accuracy": 0.9766964536122601, + "num_tokens": 7601281.0, + "step": 3250 + }, + { + "entropy": 0.12881986042484642, + "epoch": 8.209215442092155, + "grad_norm": 0.1927814930677414, + "learning_rate": 3.7479278722798766e-05, + "loss": 0.06404335975646973, + "mean_token_accuracy": 0.9769090154767036, + "num_tokens": 7720929.0, + "step": 3300 + }, + { + "entropy": 0.12998204486444592, + "epoch": 8.333748443337484, + "grad_norm": 0.24335867166519165, + "learning_rate": 3.260443148088445e-05, + "loss": 0.06666117668151855, + "mean_token_accuracy": 0.9762203323841095, + "num_tokens": 7834637.0, + "step": 3350 + }, + { + "entropy": 0.1280067889392376, + "epoch": 8.458281444582815, + "grad_norm": 0.17500700056552887, + "learning_rate": 2.8040574066167198e-05, + "loss": 0.06661289691925049, + "mean_token_accuracy": 0.9759868398308754, + "num_tokens": 7947969.0, + "step": 3400 + }, + { + "entropy": 0.12875201934948563, + "epoch": 8.582814445828145, + "grad_norm": 0.18803425133228302, + "learning_rate": 2.3796307825804367e-05, + "loss": 0.06650640487670899, + "mean_token_accuracy": 0.9768599861860275, + "num_tokens": 8063392.0, + "step": 3450 + }, + { + "entropy": 0.12174921065568924, + "epoch": 8.707347447073474, + "grad_norm": 0.11899783462285995, + "learning_rate": 1.9879631784314972e-05, + "loss": 0.06299989223480225, + "mean_token_accuracy": 0.9767305806279183, + "num_tokens": 8186140.0, + "step": 3500 + }, + { + "entropy": 0.128081704005599, + "epoch": 8.831880448318804, + "grad_norm": 0.197890043258667, + "learning_rate": 1.6297927568089764e-05, + "loss": 0.06515517234802246, + "mean_token_accuracy": 0.9760154831409454, + "num_tokens": 8304658.0, + "step": 3550 + }, + { + "entropy": 0.13355719508603214, + "epoch": 8.956413449564135, + "grad_norm": 0.18019406497478485, + "learning_rate": 1.3057945493490295e-05, + "loss": 0.06744161605834961, + "mean_token_accuracy": 0.9749321505427361, + "num_tokens": 8419672.0, + "step": 3600 + }, + { + "epoch": 9.0, + "eval_entropy": 0.23422029623112015, + "eval_loss": 0.8887823224067688, + "eval_mean_token_accuracy": 0.8543517367784367, + "eval_num_tokens": 8460261.0, + "eval_runtime": 80.4762, + "eval_samples_per_second": 17.098, + "eval_steps_per_second": 2.137, + "step": 3618 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.4901950953099264e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f4526185e7096bb5bc0e8e66ad9463a16fc5deaa --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-402/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 402, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.894650812304794e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a4461638ada4411fed26742808352c4406414c --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4020/trainer_state.json @@ -0,0 +1,944 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4020, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + }, + { + "entropy": 0.4963809953193472, + "epoch": 2.1145703611457036, + "grad_norm": 0.4585464298725128, + "learning_rate": 0.0003805303122081248, + "loss": 0.4297080230712891, + "mean_token_accuracy": 0.8698998027377658, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.49841349229216575, + "epoch": 2.2391033623910337, + "grad_norm": 0.4582020342350006, + "learning_rate": 0.00037711183154849745, + "loss": 0.4334832763671875, + "mean_token_accuracy": 0.8692125943303108, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.5070917446911335, + "epoch": 2.3636363636363638, + "grad_norm": 0.43267160654067993, + "learning_rate": 0.0003733550593128611, + "loss": 0.44077606201171876, + "mean_token_accuracy": 0.8663433310389519, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.5030459056794644, + "epoch": 2.488169364881694, + "grad_norm": 0.6674911379814148, + "learning_rate": 0.0003692670757623313, + "loss": 0.44177467346191407, + "mean_token_accuracy": 0.865137224495411, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.5059828527271748, + "epoch": 2.6127023661270234, + "grad_norm": 0.4370676279067993, + "learning_rate": 0.00036485558538074613, + "loss": 0.4437343978881836, + "mean_token_accuracy": 0.8662868231534958, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.5094366405904293, + "epoch": 2.7372353673723535, + "grad_norm": 0.4469081461429596, + "learning_rate": 0.00036012890235428757, + "loss": 0.443253173828125, + "mean_token_accuracy": 0.8655162483453751, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.5226379410922527, + "epoch": 2.8617683686176836, + "grad_norm": 0.6992693543434143, + "learning_rate": 0.0003550959349020165, + "loss": 0.4548416900634766, + "mean_token_accuracy": 0.8627148562669754, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.5179457207024097, + "epoch": 2.9863013698630136, + "grad_norm": 0.43076467514038086, + "learning_rate": 0.000349766168486855, + "loss": 0.44712478637695313, + "mean_token_accuracy": 0.8639427620172501, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5369389817811722, + "eval_loss": 0.5201929807662964, + "eval_mean_token_accuracy": 0.8542189033225526, + "eval_num_tokens": 2820087.0, + "eval_runtime": 80.0102, + "eval_samples_per_second": 17.198, + "eval_steps_per_second": 2.15, + "step": 1206 + }, + { + "entropy": 0.42975082195768455, + "epoch": 3.1095890410958904, + "grad_norm": 0.44258150458335876, + "learning_rate": 0.0003441496479386561, + "loss": 0.3603226852416992, + "mean_token_accuracy": 0.8864873057664043, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.425300100594759, + "epoch": 3.2341220423412205, + "grad_norm": 0.46550971269607544, + "learning_rate": 0.000338256958523054, + "loss": 0.3541151809692383, + "mean_token_accuracy": 0.8874222806096077, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.4290286618471146, + "epoch": 3.3586550435865505, + "grad_norm": 0.5829023718833923, + "learning_rate": 0.00033209920599177326, + "loss": 0.35764781951904295, + "mean_token_accuracy": 0.8852742439508439, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.4331103308498859, + "epoch": 3.4831880448318806, + "grad_norm": 0.494881272315979, + "learning_rate": 0.00032568799565199543, + "loss": 0.3636521530151367, + "mean_token_accuracy": 0.8851453733444213, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.42992967620491984, + "epoch": 3.6077210460772102, + "grad_norm": 0.47595155239105225, + "learning_rate": 0.00031903541049423063, + "loss": 0.36061588287353513, + "mean_token_accuracy": 0.8861182153224945, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.44519103825092315, + "epoch": 3.7322540473225407, + "grad_norm": 0.4148159623146057, + "learning_rate": 0.0003121539884199152, + "loss": 0.37374584197998045, + "mean_token_accuracy": 0.882867374420166, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.439686142206192, + "epoch": 3.8567870485678704, + "grad_norm": 0.3918115794658661, + "learning_rate": 0.0003050566986116549, + "loss": 0.3728714752197266, + "mean_token_accuracy": 0.8835265552997589, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.4370816157758236, + "epoch": 3.9813200498132004, + "grad_norm": 0.5257322192192078, + "learning_rate": 0.0002977569170906462, + "loss": 0.36963958740234376, + "mean_token_accuracy": 0.883085435628891, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48418363807506337, + "eval_loss": 0.5320115089416504, + "eval_mean_token_accuracy": 0.853746736812037, + "eval_num_tokens": 3760116.0, + "eval_runtime": 80.2791, + "eval_samples_per_second": 17.14, + "eval_steps_per_second": 2.143, + "step": 1608 + }, + { + "entropy": 0.35559599267111885, + "epoch": 4.104607721046078, + "grad_norm": 0.5824428796768188, + "learning_rate": 0.0002902684015073434, + "loss": 0.2808858108520508, + "mean_token_accuracy": 0.9084500258017068, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.34071226201951504, + "epoch": 4.229140722291407, + "grad_norm": 0.4689781069755554, + "learning_rate": 0.0002826052652128818, + "loss": 0.2676906204223633, + "mean_token_accuracy": 0.9113168820738793, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.339734458848834, + "epoch": 4.353673723536737, + "grad_norm": 0.5321401953697205, + "learning_rate": 0.0002747819506601238, + "loss": 0.26818386077880857, + "mean_token_accuracy": 0.9111532375216485, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3525573747605085, + "epoch": 4.478206724782067, + "grad_norm": 0.5021588206291199, + "learning_rate": 0.0002668132021844582, + "loss": 0.27898569107055665, + "mean_token_accuracy": 0.9079484361410141, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.35562075704336166, + "epoch": 4.602739726027397, + "grad_norm": 0.47950589656829834, + "learning_rate": 0.000258714038215652, + "loss": 0.28017072677612304, + "mean_token_accuracy": 0.9064878270030021, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.3466539917141199, + "epoch": 4.7272727272727275, + "grad_norm": 0.4677203297615051, + "learning_rate": 0.00025049972297312516, + "loss": 0.27576370239257814, + "mean_token_accuracy": 0.9086684882640839, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.34570303693413734, + "epoch": 4.851805728518057, + "grad_norm": 0.4588606357574463, + "learning_rate": 0.0002421857376979947, + "loss": 0.2746767234802246, + "mean_token_accuracy": 0.9086508122086525, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3503329714387655, + "epoch": 4.976338729763388, + "grad_norm": 0.5665764212608337, + "learning_rate": 0.00023378775147610495, + "loss": 0.27303371429443357, + "mean_token_accuracy": 0.9096201673150063, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.41260595723640087, + "eval_loss": 0.5472355484962463, + "eval_mean_token_accuracy": 0.8565515792647074, + "eval_num_tokens": 4700145.0, + "eval_runtime": 80.2872, + "eval_samples_per_second": 17.138, + "eval_steps_per_second": 2.142, + "step": 2010 + }, + { + "entropy": 0.2732633537415302, + "epoch": 5.099626400996264, + "grad_norm": 0.567966639995575, + "learning_rate": 0.0002253215917070334, + "loss": 0.19740861892700196, + "mean_token_accuracy": 0.9340365438750295, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.24825763009488583, + "epoch": 5.224159402241594, + "grad_norm": 0.6070764064788818, + "learning_rate": 0.00021680321427472769, + "loss": 0.17449329376220704, + "mean_token_accuracy": 0.9399480578303337, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.26008583076298236, + "epoch": 5.348692403486924, + "grad_norm": 0.4653824269771576, + "learning_rate": 0.00020824867347599314, + "loss": 0.1837337875366211, + "mean_token_accuracy": 0.9369171789288521, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.26238673031330106, + "epoch": 5.473225404732254, + "grad_norm": 0.5280150771141052, + "learning_rate": 0.0001996740917635043, + "loss": 0.18675100326538085, + "mean_token_accuracy": 0.936633392572403, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.26483887039124965, + "epoch": 5.597758405977584, + "grad_norm": 0.4668729305267334, + "learning_rate": 0.00019109562936036546, + "loss": 0.18501157760620118, + "mean_token_accuracy": 0.9365152379870415, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.2524403877556324, + "epoch": 5.722291407222914, + "grad_norm": 0.5033115148544312, + "learning_rate": 0.00018252945380348672, + "loss": 0.1812763023376465, + "mean_token_accuracy": 0.9381375980377197, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.25848230116069315, + "epoch": 5.846824408468244, + "grad_norm": 0.6071104407310486, + "learning_rate": 0.0001739917094731765, + "loss": 0.18280351638793946, + "mean_token_accuracy": 0.9371961706876755, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.2512618475407362, + "epoch": 5.971357409713574, + "grad_norm": 0.5169245004653931, + "learning_rate": 0.00016549848716637528, + "loss": 0.1812678337097168, + "mean_token_accuracy": 0.93798923432827, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.35272262487993683, + "eval_loss": 0.6165693998336792, + "eval_mean_token_accuracy": 0.8541817599257757, + "eval_num_tokens": 5640174.0, + "eval_runtime": 80.0797, + "eval_samples_per_second": 17.183, + "eval_steps_per_second": 2.148, + "step": 2412 + }, + { + "entropy": 0.20745287752813762, + "epoch": 6.094645080946451, + "grad_norm": 0.33874237537384033, + "learning_rate": 0.000157065793770877, + "loss": 0.1288090705871582, + "mean_token_accuracy": 0.9561566412448883, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.18590272210538386, + "epoch": 6.219178082191781, + "grad_norm": 0.43293333053588867, + "learning_rate": 0.0001487095220976913, + "loss": 0.11404885292053223, + "mean_token_accuracy": 0.961484112739563, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.18391913663595916, + "epoch": 6.34371108343711, + "grad_norm": 0.38351935148239136, + "learning_rate": 0.00014044542092840146, + "loss": 0.11131843566894531, + "mean_token_accuracy": 0.9621073800325394, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.17878302346915007, + "epoch": 6.468244084682441, + "grad_norm": 0.430851012468338, + "learning_rate": 0.00013228906533397082, + "loss": 0.10821202278137207, + "mean_token_accuracy": 0.9637334972620011, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.18323683477938174, + "epoch": 6.592777085927771, + "grad_norm": 0.3986610174179077, + "learning_rate": 0.00012425582732093497, + "loss": 0.11400936126708984, + "mean_token_accuracy": 0.9619395446777343, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.18330949898809196, + "epoch": 6.717310087173101, + "grad_norm": 0.5595681667327881, + "learning_rate": 0.0001163608468603028, + "loss": 0.113320951461792, + "mean_token_accuracy": 0.9610482943058014, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.18616238351911307, + "epoch": 6.841843088418431, + "grad_norm": 0.45847031474113464, + "learning_rate": 0.00010861900335376734, + "loss": 0.11270190238952636, + "mean_token_accuracy": 0.9607535809278488, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.17851211048662663, + "epoch": 6.966376089663761, + "grad_norm": 0.3785495162010193, + "learning_rate": 0.00010104488759100174, + "loss": 0.1098051643371582, + "mean_token_accuracy": 0.9622898560762405, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29157780076182166, + "eval_loss": 0.7041030526161194, + "eval_mean_token_accuracy": 0.8533015802156093, + "eval_num_tokens": 6580203.0, + "eval_runtime": 80.3186, + "eval_samples_per_second": 17.132, + "eval_steps_per_second": 2.141, + "step": 2814 + }, + { + "entropy": 0.16289814633100924, + "epoch": 7.089663760896638, + "grad_norm": 0.21091541647911072, + "learning_rate": 9.365277425089424e-05, + "loss": 0.08835570335388183, + "mean_token_accuracy": 0.9695430569576494, + "num_tokens": 6661238.0, + "step": 2850 + }, + { + "entropy": 0.1420974024385214, + "epoch": 7.214196762141968, + "grad_norm": 0.365073561668396, + "learning_rate": 8.645659499854436e-05, + "loss": 0.0793186902999878, + "mean_token_accuracy": 0.9731000065803528, + "num_tokens": 6774391.0, + "step": 2900 + }, + { + "entropy": 0.14309360027313234, + "epoch": 7.338729763387297, + "grad_norm": 0.388925164937973, + "learning_rate": 7.946991222872743e-05, + "loss": 0.07888932704925537, + "mean_token_accuracy": 0.9733487150073051, + "num_tokens": 6892967.0, + "step": 2950 + }, + { + "entropy": 0.14983555225655437, + "epoch": 7.463262764632628, + "grad_norm": 0.32667651772499084, + "learning_rate": 7.270589350530889e-05, + "loss": 0.07984059333801269, + "mean_token_accuracy": 0.9718177166581153, + "num_tokens": 7006692.0, + "step": 3000 + }, + { + "entropy": 0.14554835664108395, + "epoch": 7.587795765877957, + "grad_norm": 0.25567975640296936, + "learning_rate": 6.617728674478406e-05, + "loss": 0.0789554786682129, + "mean_token_accuracy": 0.9735771152377128, + "num_tokens": 7124768.0, + "step": 3050 + }, + { + "entropy": 0.1400925239548087, + "epoch": 7.712328767123288, + "grad_norm": 0.38630175590515137, + "learning_rate": 5.989639619071232e-05, + "loss": 0.078687162399292, + "mean_token_accuracy": 0.9736493360996247, + "num_tokens": 7242764.0, + "step": 3100 + }, + { + "entropy": 0.13647916626185178, + "epoch": 7.8368617683686175, + "grad_norm": 0.2967790961265564, + "learning_rate": 5.3875059224327037e-05, + "loss": 0.07509267330169678, + "mean_token_accuracy": 0.9741105973720551, + "num_tokens": 7366604.0, + "step": 3150 + }, + { + "entropy": 0.14620193544775248, + "epoch": 7.961394769613948, + "grad_norm": 0.2764667272567749, + "learning_rate": 4.812462405502509e-05, + "loss": 0.07793283462524414, + "mean_token_accuracy": 0.9722180238366127, + "num_tokens": 7481687.0, + "step": 3200 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2592288721785989, + "eval_loss": 0.7876929640769958, + "eval_mean_token_accuracy": 0.8539637528186621, + "eval_num_tokens": 7520232.0, + "eval_runtime": 80.8149, + "eval_samples_per_second": 17.027, + "eval_steps_per_second": 2.128, + "step": 3216 + }, + { + "entropy": 0.1309273194220632, + "epoch": 8.084682440846825, + "grad_norm": 0.24712106585502625, + "learning_rate": 4.265592833278151e-05, + "loss": 0.06616766929626465, + "mean_token_accuracy": 0.9766964536122601, + "num_tokens": 7601281.0, + "step": 3250 + }, + { + "entropy": 0.12881986042484642, + "epoch": 8.209215442092155, + "grad_norm": 0.1927814930677414, + "learning_rate": 3.7479278722798766e-05, + "loss": 0.06404335975646973, + "mean_token_accuracy": 0.9769090154767036, + "num_tokens": 7720929.0, + "step": 3300 + }, + { + "entropy": 0.12998204486444592, + "epoch": 8.333748443337484, + "grad_norm": 0.24335867166519165, + "learning_rate": 3.260443148088445e-05, + "loss": 0.06666117668151855, + "mean_token_accuracy": 0.9762203323841095, + "num_tokens": 7834637.0, + "step": 3350 + }, + { + "entropy": 0.1280067889392376, + "epoch": 8.458281444582815, + "grad_norm": 0.17500700056552887, + "learning_rate": 2.8040574066167198e-05, + "loss": 0.06661289691925049, + "mean_token_accuracy": 0.9759868398308754, + "num_tokens": 7947969.0, + "step": 3400 + }, + { + "entropy": 0.12875201934948563, + "epoch": 8.582814445828145, + "grad_norm": 0.18803425133228302, + "learning_rate": 2.3796307825804367e-05, + "loss": 0.06650640487670899, + "mean_token_accuracy": 0.9768599861860275, + "num_tokens": 8063392.0, + "step": 3450 + }, + { + "entropy": 0.12174921065568924, + "epoch": 8.707347447073474, + "grad_norm": 0.11899783462285995, + "learning_rate": 1.9879631784314972e-05, + "loss": 0.06299989223480225, + "mean_token_accuracy": 0.9767305806279183, + "num_tokens": 8186140.0, + "step": 3500 + }, + { + "entropy": 0.128081704005599, + "epoch": 8.831880448318804, + "grad_norm": 0.197890043258667, + "learning_rate": 1.6297927568089764e-05, + "loss": 0.06515517234802246, + "mean_token_accuracy": 0.9760154831409454, + "num_tokens": 8304658.0, + "step": 3550 + }, + { + "entropy": 0.13355719508603214, + "epoch": 8.956413449564135, + "grad_norm": 0.18019406497478485, + "learning_rate": 1.3057945493490295e-05, + "loss": 0.06744161605834961, + "mean_token_accuracy": 0.9749321505427361, + "num_tokens": 8419672.0, + "step": 3600 + }, + { + "epoch": 9.0, + "eval_entropy": 0.23422029623112015, + "eval_loss": 0.8887823224067688, + "eval_mean_token_accuracy": 0.8543517367784367, + "eval_num_tokens": 8460261.0, + "eval_runtime": 80.4762, + "eval_samples_per_second": 17.098, + "eval_steps_per_second": 2.137, + "step": 3618 + }, + { + "entropy": 0.12957414030802972, + "epoch": 9.079701120797012, + "grad_norm": 0.12944698333740234, + "learning_rate": 1.0165791844757058e-05, + "loss": 0.06388524055480957, + "mean_token_accuracy": 0.9766391768600001, + "num_tokens": 8531214.0, + "step": 3650 + }, + { + "entropy": 0.11808951755985618, + "epoch": 9.204234122042342, + "grad_norm": 0.13890483975410461, + "learning_rate": 7.626917365702756e-06, + "loss": 0.05949122905731201, + "mean_token_accuracy": 0.9785410597920418, + "num_tokens": 8652483.0, + "step": 3700 + }, + { + "entropy": 0.1274303700402379, + "epoch": 9.32876712328767, + "grad_norm": 0.10332873463630676, + "learning_rate": 5.44610698688034e-06, + "loss": 0.06253890991210938, + "mean_token_accuracy": 0.9769675573706627, + "num_tokens": 8767142.0, + "step": 3750 + }, + { + "entropy": 0.11402286102995277, + "epoch": 9.453300124533001, + "grad_norm": 0.12605386972427368, + "learning_rate": 3.627470807587018e-06, + "loss": 0.05756766319274902, + "mean_token_accuracy": 0.9795342680811882, + "num_tokens": 8889508.0, + "step": 3800 + }, + { + "entropy": 0.12327373247593641, + "epoch": 9.577833125778332, + "grad_norm": 0.13971152901649475, + "learning_rate": 2.1744363496994704e-06, + "loss": 0.06330667495727539, + "mean_token_accuracy": 0.9768437051773071, + "num_tokens": 9000626.0, + "step": 3850 + }, + { + "entropy": 0.12451133135706187, + "epoch": 9.70236612702366, + "grad_norm": 0.13401846587657928, + "learning_rate": 1.0897420979397859e-06, + "loss": 0.060658202171325684, + "mean_token_accuracy": 0.9773744866251945, + "num_tokens": 9118844.0, + "step": 3900 + }, + { + "entropy": 0.11562174517661333, + "epoch": 9.826899128268991, + "grad_norm": 0.11007044464349747, + "learning_rate": 3.7543233874618285e-07, + "loss": 0.059203782081604005, + "mean_token_accuracy": 0.9786800450086593, + "num_tokens": 9240716.0, + "step": 3950 + }, + { + "entropy": 0.1279593655653298, + "epoch": 9.951432129514322, + "grad_norm": 0.1578713208436966, + "learning_rate": 3.285330747582018e-08, + "loss": 0.06231212615966797, + "mean_token_accuracy": 0.9761605298519135, + "num_tokens": 9354788.0, + "step": 4000 + }, + { + "epoch": 10.0, + "eval_entropy": 0.23064094478654307, + "eval_loss": 0.9308294057846069, + "eval_mean_token_accuracy": 0.8536452148542848, + "eval_num_tokens": 9400290.0, + "eval_runtime": 80.0259, + "eval_samples_per_second": 17.194, + "eval_steps_per_second": 2.149, + "step": 4020 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3.8801038188323635e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9bde257ac079039d07b5273a88438c0c0f378c8 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04293738824192794, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c31fe5a61963d343feea2b3021c122e4ef90ea76 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-804/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 804, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.4945576781034469, + "epoch": 0.12453300124533001, + "grad_norm": 0.7291187047958374, + "learning_rate": 4.8174802250282295e-05, + "loss": 1.3015669250488282, + "mean_token_accuracy": 0.7135214237868786, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7246888878941536, + "epoch": 0.24906600249066002, + "grad_norm": 0.5772781372070312, + "learning_rate": 9.733276373016218e-05, + "loss": 0.642754135131836, + "mean_token_accuracy": 0.8242821827530861, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838392898440361, + "epoch": 0.37359900373599003, + "grad_norm": 0.4728560149669647, + "learning_rate": 0.00014649072521004207, + "loss": 0.6014968490600586, + "mean_token_accuracy": 0.8303881451487541, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6594706049561501, + "epoch": 0.49813200498132004, + "grad_norm": 0.5458584427833557, + "learning_rate": 0.00019564868668992197, + "loss": 0.5789081954956055, + "mean_token_accuracy": 0.8367286545038223, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6431240077316761, + "epoch": 0.6226650062266501, + "grad_norm": 0.6594698429107666, + "learning_rate": 0.0002448066481698019, + "loss": 0.562723274230957, + "mean_token_accuracy": 0.8406605333089828, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6416522780060768, + "epoch": 0.7471980074719801, + "grad_norm": 0.3835357427597046, + "learning_rate": 0.0002939646096496817, + "loss": 0.5631664657592773, + "mean_token_accuracy": 0.8386288139224053, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6282059556245804, + "epoch": 0.8717310087173101, + "grad_norm": 0.4643607437610626, + "learning_rate": 0.0003431225711295616, + "loss": 0.5579532241821289, + "mean_token_accuracy": 0.8415955656766891, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6296975083649159, + "epoch": 0.9962640099626401, + "grad_norm": 0.5332581400871277, + "learning_rate": 0.00039228053260944153, + "loss": 0.5614005279541016, + "mean_token_accuracy": 0.8401218938827515, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6447091469931048, + "eval_loss": 0.5722090005874634, + "eval_mean_token_accuracy": 0.8409275641275007, + "eval_num_tokens": 940029.0, + "eval_runtime": 81.0912, + "eval_samples_per_second": 16.969, + "eval_steps_per_second": 2.121, + "step": 402 + }, + { + "entropy": 0.592438119981024, + "epoch": 1.1195516811955168, + "grad_norm": 0.5156843066215515, + "learning_rate": 0.00039506546405195707, + "loss": 0.5288405227661133, + "mean_token_accuracy": 0.8473661833941334, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.6035540834069252, + "epoch": 1.244084682440847, + "grad_norm": 0.6018164753913879, + "learning_rate": 0.00039452946012175837, + "loss": 0.5313962936401367, + "mean_token_accuracy": 0.8469011873006821, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.6012892751395702, + "epoch": 1.3686176836861768, + "grad_norm": 0.5239841341972351, + "learning_rate": 0.0003936223382021896, + "loss": 0.5329293823242187, + "mean_token_accuracy": 0.8464742597937583, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.597025256305933, + "epoch": 1.4931506849315068, + "grad_norm": 0.4814896285533905, + "learning_rate": 0.00039234580791514196, + "loss": 0.5313105010986328, + "mean_token_accuracy": 0.8474235320091248, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.594612804800272, + "epoch": 1.6176836861768369, + "grad_norm": 0.6025446057319641, + "learning_rate": 0.0003907022750939179, + "loss": 0.529390640258789, + "mean_token_accuracy": 0.8483339309692383, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5792807288467884, + "epoch": 1.7422166874221667, + "grad_norm": 0.3983995318412781, + "learning_rate": 0.00038869483724903863, + "loss": 0.5156287384033204, + "mean_token_accuracy": 0.8519385030865669, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5724843810498714, + "epoch": 1.8667496886674968, + "grad_norm": 0.6686900854110718, + "learning_rate": 0.0003863272777304715, + "loss": 0.5083164978027344, + "mean_token_accuracy": 0.8507362252473831, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5764366070926189, + "epoch": 1.9912826899128269, + "grad_norm": 0.47789353132247925, + "learning_rate": 0.0003836040585972784, + "loss": 0.5066570663452148, + "mean_token_accuracy": 0.8517848494648933, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5557634790622911, + "eval_loss": 0.5285266637802124, + "eval_mean_token_accuracy": 0.8505693410718164, + "eval_num_tokens": 1880058.0, + "eval_runtime": 80.5803, + "eval_samples_per_second": 17.076, + "eval_steps_per_second": 2.135, + "step": 804 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.764884851111526e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2010/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2010/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..773562d3c8647e91118ec12374c8dba856868541 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2010/trainer_state.json @@ -0,0 +1,489 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2010, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + }, + { + "entropy": 0.5661561889479859, + "epoch": 1.1195516811955168, + "grad_norm": 0.708380937576294, + "learning_rate": 0.00018411093929332744, + "loss": 0.4969264221191406, + "mean_token_accuracy": 0.8550105146085373, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.5690013617277145, + "epoch": 1.244084682440847, + "grad_norm": 0.8610317707061768, + "learning_rate": 0.00018386114730685091, + "loss": 0.49536712646484377, + "mean_token_accuracy": 0.8549688473343849, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.5661576768755913, + "epoch": 1.3686176836861768, + "grad_norm": 0.8410939574241638, + "learning_rate": 0.00018343840453669726, + "loss": 0.4957312774658203, + "mean_token_accuracy": 0.8546139162778854, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.5658727620542049, + "epoch": 1.4931506849315068, + "grad_norm": 0.6218363046646118, + "learning_rate": 0.00018284350771181608, + "loss": 0.4974911499023438, + "mean_token_accuracy": 0.855501549243927, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.5707372877001763, + "epoch": 1.6176836861768369, + "grad_norm": 0.8585225343704224, + "learning_rate": 0.0001820775780140605, + "loss": 0.49875095367431643, + "mean_token_accuracy": 0.8563189420104027, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5534089787304401, + "epoch": 1.7422166874221667, + "grad_norm": 0.6285793781280518, + "learning_rate": 0.00018114205896513374, + "loss": 0.48918685913085935, + "mean_token_accuracy": 0.8578527864813804, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5458812513947486, + "epoch": 1.8667496886674968, + "grad_norm": 0.71883225440979, + "learning_rate": 0.00018003871370603276, + "loss": 0.482542839050293, + "mean_token_accuracy": 0.8564121940732002, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5502609372138977, + "epoch": 1.9912826899128269, + "grad_norm": 0.6442635655403137, + "learning_rate": 0.00017876962167411625, + "loss": 0.48193965911865233, + "mean_token_accuracy": 0.8567470651865006, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5426353398104047, + "eval_loss": 0.5083975791931152, + "eval_mean_token_accuracy": 0.8554168769786524, + "eval_num_tokens": 1880058.0, + "eval_runtime": 79.8868, + "eval_samples_per_second": 17.212, + "eval_steps_per_second": 2.153, + "step": 804 + }, + { + "entropy": 0.4734916226430373, + "epoch": 2.1145703611457036, + "grad_norm": 0.7688794732093811, + "learning_rate": 0.00017733717468405966, + "loss": 0.40314048767089844, + "mean_token_accuracy": 0.8765652631268357, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.46802927538752553, + "epoch": 2.2391033623910337, + "grad_norm": 0.6301783919334412, + "learning_rate": 0.00017574407242008323, + "loss": 0.3979022979736328, + "mean_token_accuracy": 0.8784176242351532, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.4625655211508274, + "epoch": 2.3636363636363638, + "grad_norm": 0.5972099900245667, + "learning_rate": 0.00017399331734794884, + "loss": 0.4014426803588867, + "mean_token_accuracy": 0.8762527570128441, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.470507246106863, + "epoch": 2.488169364881694, + "grad_norm": 0.8060091733932495, + "learning_rate": 0.00017208820905631456, + "loss": 0.4042459869384766, + "mean_token_accuracy": 0.8747306689620018, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.4665305399894714, + "epoch": 2.6127023661270234, + "grad_norm": 0.6650318503379822, + "learning_rate": 0.0001700323380381121, + "loss": 0.40335521697998045, + "mean_token_accuracy": 0.8762387800216674, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.46196050629019736, + "epoch": 2.7372353673723535, + "grad_norm": 0.8204917907714844, + "learning_rate": 0.00016782957892366653, + "loss": 0.4012960052490234, + "mean_token_accuracy": 0.8761517608165741, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.4787097446620464, + "epoch": 2.8617683686176836, + "grad_norm": 0.6445292830467224, + "learning_rate": 0.00016548408317831203, + "loss": 0.4159215545654297, + "mean_token_accuracy": 0.8722453466057778, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.4758198526501656, + "epoch": 2.9863013698630136, + "grad_norm": 0.7560163736343384, + "learning_rate": 0.0001630002712782661, + "loss": 0.40826087951660156, + "mean_token_accuracy": 0.873089083135128, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.49463067810202754, + "eval_loss": 0.5086550116539001, + "eval_mean_token_accuracy": 0.857228794416716, + "eval_num_tokens": 2820087.0, + "eval_runtime": 79.7846, + "eval_samples_per_second": 17.234, + "eval_steps_per_second": 2.156, + "step": 1206 + }, + { + "entropy": 0.3859369834565153, + "epoch": 3.1095890410958904, + "grad_norm": 0.741679847240448, + "learning_rate": 0.00016038282437950814, + "loss": 0.32163028717041015, + "mean_token_accuracy": 0.8972690379378772, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.3825602190196514, + "epoch": 3.2341220423412205, + "grad_norm": 0.8756065368652344, + "learning_rate": 0.00015763667549536354, + "loss": 0.316363639831543, + "mean_token_accuracy": 0.8986149084568024, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.38129442662000657, + "epoch": 3.3586550435865505, + "grad_norm": 0.8529497981071472, + "learning_rate": 0.00015476700019942104, + "loss": 0.3164650917053223, + "mean_token_accuracy": 0.8971891248226166, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.3877453975379467, + "epoch": 3.4831880448318806, + "grad_norm": 0.7949019074440002, + "learning_rate": 0.0001517792068713048, + "loss": 0.31930156707763674, + "mean_token_accuracy": 0.8973806884884834, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.38809079021215437, + "epoch": 3.6077210460772102, + "grad_norm": 0.8050806522369385, + "learning_rate": 0.0001486789265036849, + "loss": 0.31977331161499023, + "mean_token_accuracy": 0.8985486030578613, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.3984291091561317, + "epoch": 3.7322540473225407, + "grad_norm": 0.7165534496307373, + "learning_rate": 0.00014547200208973657, + "loss": 0.32611679077148437, + "mean_token_accuracy": 0.895875141620636, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.39771562442183495, + "epoch": 3.8567870485678704, + "grad_norm": 0.6194931864738464, + "learning_rate": 0.00014216447761104936, + "loss": 0.32970951080322264, + "mean_token_accuracy": 0.8950823175907135, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.3997128288447857, + "epoch": 3.9813200498132004, + "grad_norm": 0.7835033535957336, + "learning_rate": 0.00013876258664674012, + "loss": 0.3263474655151367, + "mean_token_accuracy": 0.894331750869751, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45059218794800515, + "eval_loss": 0.5254963636398315, + "eval_mean_token_accuracy": 0.857301191535107, + "eval_num_tokens": 3760116.0, + "eval_runtime": 79.3533, + "eval_samples_per_second": 17.328, + "eval_steps_per_second": 2.168, + "step": 1608 + }, + { + "entropy": 0.31803849610415375, + "epoch": 4.104607721046078, + "grad_norm": 0.9932603240013123, + "learning_rate": 0.00013527274062523803, + "loss": 0.24486299514770507, + "mean_token_accuracy": 0.9204365689345081, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.30566568687558177, + "epoch": 4.229140722291407, + "grad_norm": 0.8069589138031006, + "learning_rate": 0.0001317015167408831, + "loss": 0.22590700149536133, + "mean_token_accuracy": 0.9249501618742942, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.2980187537521124, + "epoch": 4.353673723536737, + "grad_norm": 0.6587982773780823, + "learning_rate": 0.00012805564555811122, + "loss": 0.22620832443237304, + "mean_token_accuracy": 0.9248810124397278, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3055800000578165, + "epoch": 4.478206724782067, + "grad_norm": 0.8378816843032837, + "learning_rate": 0.00012434199832658783, + "loss": 0.23171533584594728, + "mean_token_accuracy": 0.9231083118915557, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.31550996117293834, + "epoch": 4.602739726027397, + "grad_norm": 0.749613881111145, + "learning_rate": 0.00012056757403119691, + "loss": 0.23747362136840822, + "mean_token_accuracy": 0.9206199425458909, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.30942291468381883, + "epoch": 4.7272727272727275, + "grad_norm": 0.7933925986289978, + "learning_rate": 0.0001167394862012918, + "loss": 0.23256269454956055, + "mean_token_accuracy": 0.9226748991012573, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.3098080876469612, + "epoch": 4.851805728518057, + "grad_norm": 0.8441582918167114, + "learning_rate": 0.00011286494950406773, + "loss": 0.23220256805419923, + "mean_token_accuracy": 0.9222584775090218, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3098256988823414, + "epoch": 4.976338729763388, + "grad_norm": 0.8682230114936829, + "learning_rate": 0.00010895126614732359, + "loss": 0.23128936767578126, + "mean_token_accuracy": 0.923048702776432, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.39352814238085304, + "eval_loss": 0.5585312247276306, + "eval_mean_token_accuracy": 0.8575408472571262, + "eval_num_tokens": 4700145.0, + "eval_runtime": 79.6426, + "eval_samples_per_second": 17.265, + "eval_steps_per_second": 2.16, + "step": 2010 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9406889736070554e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd48592ebfde6ffa9e1fa29412808c550e6c8898 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05833860659003597, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..50d900a9fcac9aae8527b238bb08019169c228a2 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2412/trainer_state.json @@ -0,0 +1,580 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2412, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + }, + { + "entropy": 0.5661561889479859, + "epoch": 1.1195516811955168, + "grad_norm": 0.708380937576294, + "learning_rate": 0.00018411093929332744, + "loss": 0.4969264221191406, + "mean_token_accuracy": 0.8550105146085373, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.5690013617277145, + "epoch": 1.244084682440847, + "grad_norm": 0.8610317707061768, + "learning_rate": 0.00018386114730685091, + "loss": 0.49536712646484377, + "mean_token_accuracy": 0.8549688473343849, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.5661576768755913, + "epoch": 1.3686176836861768, + "grad_norm": 0.8410939574241638, + "learning_rate": 0.00018343840453669726, + "loss": 0.4957312774658203, + "mean_token_accuracy": 0.8546139162778854, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.5658727620542049, + "epoch": 1.4931506849315068, + "grad_norm": 0.6218363046646118, + "learning_rate": 0.00018284350771181608, + "loss": 0.4974911499023438, + "mean_token_accuracy": 0.855501549243927, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.5707372877001763, + "epoch": 1.6176836861768369, + "grad_norm": 0.8585225343704224, + "learning_rate": 0.0001820775780140605, + "loss": 0.49875095367431643, + "mean_token_accuracy": 0.8563189420104027, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5534089787304401, + "epoch": 1.7422166874221667, + "grad_norm": 0.6285793781280518, + "learning_rate": 0.00018114205896513374, + "loss": 0.48918685913085935, + "mean_token_accuracy": 0.8578527864813804, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5458812513947486, + "epoch": 1.8667496886674968, + "grad_norm": 0.71883225440979, + "learning_rate": 0.00018003871370603276, + "loss": 0.482542839050293, + "mean_token_accuracy": 0.8564121940732002, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5502609372138977, + "epoch": 1.9912826899128269, + "grad_norm": 0.6442635655403137, + "learning_rate": 0.00017876962167411625, + "loss": 0.48193965911865233, + "mean_token_accuracy": 0.8567470651865006, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5426353398104047, + "eval_loss": 0.5083975791931152, + "eval_mean_token_accuracy": 0.8554168769786524, + "eval_num_tokens": 1880058.0, + "eval_runtime": 79.8868, + "eval_samples_per_second": 17.212, + "eval_steps_per_second": 2.153, + "step": 804 + }, + { + "entropy": 0.4734916226430373, + "epoch": 2.1145703611457036, + "grad_norm": 0.7688794732093811, + "learning_rate": 0.00017733717468405966, + "loss": 0.40314048767089844, + "mean_token_accuracy": 0.8765652631268357, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.46802927538752553, + "epoch": 2.2391033623910337, + "grad_norm": 0.6301783919334412, + "learning_rate": 0.00017574407242008323, + "loss": 0.3979022979736328, + "mean_token_accuracy": 0.8784176242351532, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.4625655211508274, + "epoch": 2.3636363636363638, + "grad_norm": 0.5972099900245667, + "learning_rate": 0.00017399331734794884, + "loss": 0.4014426803588867, + "mean_token_accuracy": 0.8762527570128441, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.470507246106863, + "epoch": 2.488169364881694, + "grad_norm": 0.8060091733932495, + "learning_rate": 0.00017208820905631456, + "loss": 0.4042459869384766, + "mean_token_accuracy": 0.8747306689620018, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.4665305399894714, + "epoch": 2.6127023661270234, + "grad_norm": 0.6650318503379822, + "learning_rate": 0.0001700323380381121, + "loss": 0.40335521697998045, + "mean_token_accuracy": 0.8762387800216674, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.46196050629019736, + "epoch": 2.7372353673723535, + "grad_norm": 0.8204917907714844, + "learning_rate": 0.00016782957892366653, + "loss": 0.4012960052490234, + "mean_token_accuracy": 0.8761517608165741, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.4787097446620464, + "epoch": 2.8617683686176836, + "grad_norm": 0.6445292830467224, + "learning_rate": 0.00016548408317831203, + "loss": 0.4159215545654297, + "mean_token_accuracy": 0.8722453466057778, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.4758198526501656, + "epoch": 2.9863013698630136, + "grad_norm": 0.7560163736343384, + "learning_rate": 0.0001630002712782661, + "loss": 0.40826087951660156, + "mean_token_accuracy": 0.873089083135128, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.49463067810202754, + "eval_loss": 0.5086550116539001, + "eval_mean_token_accuracy": 0.857228794416716, + "eval_num_tokens": 2820087.0, + "eval_runtime": 79.7846, + "eval_samples_per_second": 17.234, + "eval_steps_per_second": 2.156, + "step": 1206 + }, + { + "entropy": 0.3859369834565153, + "epoch": 3.1095890410958904, + "grad_norm": 0.741679847240448, + "learning_rate": 0.00016038282437950814, + "loss": 0.32163028717041015, + "mean_token_accuracy": 0.8972690379378772, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.3825602190196514, + "epoch": 3.2341220423412205, + "grad_norm": 0.8756065368652344, + "learning_rate": 0.00015763667549536354, + "loss": 0.316363639831543, + "mean_token_accuracy": 0.8986149084568024, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.38129442662000657, + "epoch": 3.3586550435865505, + "grad_norm": 0.8529497981071472, + "learning_rate": 0.00015476700019942104, + "loss": 0.3164650917053223, + "mean_token_accuracy": 0.8971891248226166, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.3877453975379467, + "epoch": 3.4831880448318806, + "grad_norm": 0.7949019074440002, + "learning_rate": 0.0001517792068713048, + "loss": 0.31930156707763674, + "mean_token_accuracy": 0.8973806884884834, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.38809079021215437, + "epoch": 3.6077210460772102, + "grad_norm": 0.8050806522369385, + "learning_rate": 0.0001486789265036849, + "loss": 0.31977331161499023, + "mean_token_accuracy": 0.8985486030578613, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.3984291091561317, + "epoch": 3.7322540473225407, + "grad_norm": 0.7165534496307373, + "learning_rate": 0.00014547200208973657, + "loss": 0.32611679077148437, + "mean_token_accuracy": 0.895875141620636, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.39771562442183495, + "epoch": 3.8567870485678704, + "grad_norm": 0.6194931864738464, + "learning_rate": 0.00014216447761104936, + "loss": 0.32970951080322264, + "mean_token_accuracy": 0.8950823175907135, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.3997128288447857, + "epoch": 3.9813200498132004, + "grad_norm": 0.7835033535957336, + "learning_rate": 0.00013876258664674012, + "loss": 0.3263474655151367, + "mean_token_accuracy": 0.894331750869751, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45059218794800515, + "eval_loss": 0.5254963636398315, + "eval_mean_token_accuracy": 0.857301191535107, + "eval_num_tokens": 3760116.0, + "eval_runtime": 79.3533, + "eval_samples_per_second": 17.328, + "eval_steps_per_second": 2.168, + "step": 1608 + }, + { + "entropy": 0.31803849610415375, + "epoch": 4.104607721046078, + "grad_norm": 0.9932603240013123, + "learning_rate": 0.00013527274062523803, + "loss": 0.24486299514770507, + "mean_token_accuracy": 0.9204365689345081, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.30566568687558177, + "epoch": 4.229140722291407, + "grad_norm": 0.8069589138031006, + "learning_rate": 0.0001317015167408831, + "loss": 0.22590700149536133, + "mean_token_accuracy": 0.9249501618742942, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.2980187537521124, + "epoch": 4.353673723536737, + "grad_norm": 0.6587982773780823, + "learning_rate": 0.00012805564555811122, + "loss": 0.22620832443237304, + "mean_token_accuracy": 0.9248810124397278, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3055800000578165, + "epoch": 4.478206724782067, + "grad_norm": 0.8378816843032837, + "learning_rate": 0.00012434199832658783, + "loss": 0.23171533584594728, + "mean_token_accuracy": 0.9231083118915557, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.31550996117293834, + "epoch": 4.602739726027397, + "grad_norm": 0.749613881111145, + "learning_rate": 0.00012056757403119691, + "loss": 0.23747362136840822, + "mean_token_accuracy": 0.9206199425458909, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.30942291468381883, + "epoch": 4.7272727272727275, + "grad_norm": 0.7933925986289978, + "learning_rate": 0.0001167394862012918, + "loss": 0.23256269454956055, + "mean_token_accuracy": 0.9226748991012573, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.3098080876469612, + "epoch": 4.851805728518057, + "grad_norm": 0.8441582918167114, + "learning_rate": 0.00011286494950406773, + "loss": 0.23220256805419923, + "mean_token_accuracy": 0.9222584775090218, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3098256988823414, + "epoch": 4.976338729763388, + "grad_norm": 0.8682230114936829, + "learning_rate": 0.00010895126614732359, + "loss": 0.23128936767578126, + "mean_token_accuracy": 0.923048702776432, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.39352814238085304, + "eval_loss": 0.5585312247276306, + "eval_mean_token_accuracy": 0.8575408472571262, + "eval_num_tokens": 4700145.0, + "eval_runtime": 79.6426, + "eval_samples_per_second": 17.265, + "eval_steps_per_second": 2.16, + "step": 2010 + }, + { + "entropy": 0.242041623765471, + "epoch": 5.099626400996264, + "grad_norm": 0.8845140337944031, + "learning_rate": 0.00010500581211723872, + "loss": 0.16415899276733398, + "mean_token_accuracy": 0.9458679586949975, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.22115826569497585, + "epoch": 5.224159402241594, + "grad_norm": 0.8571121692657471, + "learning_rate": 0.00010103602327710202, + "loss": 0.14506815910339355, + "mean_token_accuracy": 0.9509596079587936, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.2277975185215473, + "epoch": 5.348692403486924, + "grad_norm": 0.6461961269378662, + "learning_rate": 9.7049381353193e-05, + "loss": 0.15208842277526854, + "mean_token_accuracy": 0.9484275713562965, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.22508278906345366, + "epoch": 5.473225404732254, + "grad_norm": 0.6664603352546692, + "learning_rate": 9.305339983422608e-05, + "loss": 0.1507566452026367, + "mean_token_accuracy": 0.9494482669234275, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.22525668181478978, + "epoch": 5.597758405977584, + "grad_norm": 0.6928425431251526, + "learning_rate": 8.905560981093345e-05, + "loss": 0.15167025566101075, + "mean_token_accuracy": 0.947735041975975, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.21739320732653142, + "epoch": 5.722291407222914, + "grad_norm": 0.7611718773841858, + "learning_rate": 8.506354578247393e-05, + "loss": 0.1465705680847168, + "mean_token_accuracy": 0.9501141354441642, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.2239777410775423, + "epoch": 5.846824408468244, + "grad_norm": 0.7929556965827942, + "learning_rate": 8.108473145641841e-05, + "loss": 0.14866115570068358, + "mean_token_accuracy": 0.9497014331817627, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.21606657084077596, + "epoch": 5.971357409713574, + "grad_norm": 0.7851046323776245, + "learning_rate": 7.712666556907332e-05, + "loss": 0.14777716636657715, + "mean_token_accuracy": 0.9501199626922607, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.32871103122137313, + "eval_loss": 0.635863184928894, + "eval_mean_token_accuracy": 0.8554029128579206, + "eval_num_tokens": 5640174.0, + "eval_runtime": 79.4073, + "eval_samples_per_second": 17.316, + "eval_steps_per_second": 2.166, + "step": 2412 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.3282680816735027e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd48592ebfde6ffa9e1fa29412808c550e6c8898 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05833860659003597, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ac7931ff2d677518a7d58e6d7e2b3d60ad97a40a --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2814/trainer_state.json @@ -0,0 +1,671 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2814, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + }, + { + "entropy": 0.5661561889479859, + "epoch": 1.1195516811955168, + "grad_norm": 0.708380937576294, + "learning_rate": 0.00018411093929332744, + "loss": 0.4969264221191406, + "mean_token_accuracy": 0.8550105146085373, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.5690013617277145, + "epoch": 1.244084682440847, + "grad_norm": 0.8610317707061768, + "learning_rate": 0.00018386114730685091, + "loss": 0.49536712646484377, + "mean_token_accuracy": 0.8549688473343849, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.5661576768755913, + "epoch": 1.3686176836861768, + "grad_norm": 0.8410939574241638, + "learning_rate": 0.00018343840453669726, + "loss": 0.4957312774658203, + "mean_token_accuracy": 0.8546139162778854, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.5658727620542049, + "epoch": 1.4931506849315068, + "grad_norm": 0.6218363046646118, + "learning_rate": 0.00018284350771181608, + "loss": 0.4974911499023438, + "mean_token_accuracy": 0.855501549243927, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.5707372877001763, + "epoch": 1.6176836861768369, + "grad_norm": 0.8585225343704224, + "learning_rate": 0.0001820775780140605, + "loss": 0.49875095367431643, + "mean_token_accuracy": 0.8563189420104027, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5534089787304401, + "epoch": 1.7422166874221667, + "grad_norm": 0.6285793781280518, + "learning_rate": 0.00018114205896513374, + "loss": 0.48918685913085935, + "mean_token_accuracy": 0.8578527864813804, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5458812513947486, + "epoch": 1.8667496886674968, + "grad_norm": 0.71883225440979, + "learning_rate": 0.00018003871370603276, + "loss": 0.482542839050293, + "mean_token_accuracy": 0.8564121940732002, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5502609372138977, + "epoch": 1.9912826899128269, + "grad_norm": 0.6442635655403137, + "learning_rate": 0.00017876962167411625, + "loss": 0.48193965911865233, + "mean_token_accuracy": 0.8567470651865006, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5426353398104047, + "eval_loss": 0.5083975791931152, + "eval_mean_token_accuracy": 0.8554168769786524, + "eval_num_tokens": 1880058.0, + "eval_runtime": 79.8868, + "eval_samples_per_second": 17.212, + "eval_steps_per_second": 2.153, + "step": 804 + }, + { + "entropy": 0.4734916226430373, + "epoch": 2.1145703611457036, + "grad_norm": 0.7688794732093811, + "learning_rate": 0.00017733717468405966, + "loss": 0.40314048767089844, + "mean_token_accuracy": 0.8765652631268357, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.46802927538752553, + "epoch": 2.2391033623910337, + "grad_norm": 0.6301783919334412, + "learning_rate": 0.00017574407242008323, + "loss": 0.3979022979736328, + "mean_token_accuracy": 0.8784176242351532, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.4625655211508274, + "epoch": 2.3636363636363638, + "grad_norm": 0.5972099900245667, + "learning_rate": 0.00017399331734794884, + "loss": 0.4014426803588867, + "mean_token_accuracy": 0.8762527570128441, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.470507246106863, + "epoch": 2.488169364881694, + "grad_norm": 0.8060091733932495, + "learning_rate": 0.00017208820905631456, + "loss": 0.4042459869384766, + "mean_token_accuracy": 0.8747306689620018, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.4665305399894714, + "epoch": 2.6127023661270234, + "grad_norm": 0.6650318503379822, + "learning_rate": 0.0001700323380381121, + "loss": 0.40335521697998045, + "mean_token_accuracy": 0.8762387800216674, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.46196050629019736, + "epoch": 2.7372353673723535, + "grad_norm": 0.8204917907714844, + "learning_rate": 0.00016782957892366653, + "loss": 0.4012960052490234, + "mean_token_accuracy": 0.8761517608165741, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.4787097446620464, + "epoch": 2.8617683686176836, + "grad_norm": 0.6445292830467224, + "learning_rate": 0.00016548408317831203, + "loss": 0.4159215545654297, + "mean_token_accuracy": 0.8722453466057778, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.4758198526501656, + "epoch": 2.9863013698630136, + "grad_norm": 0.7560163736343384, + "learning_rate": 0.0001630002712782661, + "loss": 0.40826087951660156, + "mean_token_accuracy": 0.873089083135128, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.49463067810202754, + "eval_loss": 0.5086550116539001, + "eval_mean_token_accuracy": 0.857228794416716, + "eval_num_tokens": 2820087.0, + "eval_runtime": 79.7846, + "eval_samples_per_second": 17.234, + "eval_steps_per_second": 2.156, + "step": 1206 + }, + { + "entropy": 0.3859369834565153, + "epoch": 3.1095890410958904, + "grad_norm": 0.741679847240448, + "learning_rate": 0.00016038282437950814, + "loss": 0.32163028717041015, + "mean_token_accuracy": 0.8972690379378772, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.3825602190196514, + "epoch": 3.2341220423412205, + "grad_norm": 0.8756065368652344, + "learning_rate": 0.00015763667549536354, + "loss": 0.316363639831543, + "mean_token_accuracy": 0.8986149084568024, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.38129442662000657, + "epoch": 3.3586550435865505, + "grad_norm": 0.8529497981071472, + "learning_rate": 0.00015476700019942104, + "loss": 0.3164650917053223, + "mean_token_accuracy": 0.8971891248226166, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.3877453975379467, + "epoch": 3.4831880448318806, + "grad_norm": 0.7949019074440002, + "learning_rate": 0.0001517792068713048, + "loss": 0.31930156707763674, + "mean_token_accuracy": 0.8973806884884834, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.38809079021215437, + "epoch": 3.6077210460772102, + "grad_norm": 0.8050806522369385, + "learning_rate": 0.0001486789265036849, + "loss": 0.31977331161499023, + "mean_token_accuracy": 0.8985486030578613, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.3984291091561317, + "epoch": 3.7322540473225407, + "grad_norm": 0.7165534496307373, + "learning_rate": 0.00014547200208973657, + "loss": 0.32611679077148437, + "mean_token_accuracy": 0.895875141620636, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.39771562442183495, + "epoch": 3.8567870485678704, + "grad_norm": 0.6194931864738464, + "learning_rate": 0.00014216447761104936, + "loss": 0.32970951080322264, + "mean_token_accuracy": 0.8950823175907135, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.3997128288447857, + "epoch": 3.9813200498132004, + "grad_norm": 0.7835033535957336, + "learning_rate": 0.00013876258664674012, + "loss": 0.3263474655151367, + "mean_token_accuracy": 0.894331750869751, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45059218794800515, + "eval_loss": 0.5254963636398315, + "eval_mean_token_accuracy": 0.857301191535107, + "eval_num_tokens": 3760116.0, + "eval_runtime": 79.3533, + "eval_samples_per_second": 17.328, + "eval_steps_per_second": 2.168, + "step": 1608 + }, + { + "entropy": 0.31803849610415375, + "epoch": 4.104607721046078, + "grad_norm": 0.9932603240013123, + "learning_rate": 0.00013527274062523803, + "loss": 0.24486299514770507, + "mean_token_accuracy": 0.9204365689345081, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.30566568687558177, + "epoch": 4.229140722291407, + "grad_norm": 0.8069589138031006, + "learning_rate": 0.0001317015167408831, + "loss": 0.22590700149536133, + "mean_token_accuracy": 0.9249501618742942, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.2980187537521124, + "epoch": 4.353673723536737, + "grad_norm": 0.6587982773780823, + "learning_rate": 0.00012805564555811122, + "loss": 0.22620832443237304, + "mean_token_accuracy": 0.9248810124397278, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3055800000578165, + "epoch": 4.478206724782067, + "grad_norm": 0.8378816843032837, + "learning_rate": 0.00012434199832658783, + "loss": 0.23171533584594728, + "mean_token_accuracy": 0.9231083118915557, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.31550996117293834, + "epoch": 4.602739726027397, + "grad_norm": 0.749613881111145, + "learning_rate": 0.00012056757403119691, + "loss": 0.23747362136840822, + "mean_token_accuracy": 0.9206199425458909, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.30942291468381883, + "epoch": 4.7272727272727275, + "grad_norm": 0.7933925986289978, + "learning_rate": 0.0001167394862012918, + "loss": 0.23256269454956055, + "mean_token_accuracy": 0.9226748991012573, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.3098080876469612, + "epoch": 4.851805728518057, + "grad_norm": 0.8441582918167114, + "learning_rate": 0.00011286494950406773, + "loss": 0.23220256805419923, + "mean_token_accuracy": 0.9222584775090218, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3098256988823414, + "epoch": 4.976338729763388, + "grad_norm": 0.8682230114936829, + "learning_rate": 0.00010895126614732359, + "loss": 0.23128936767578126, + "mean_token_accuracy": 0.923048702776432, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.39352814238085304, + "eval_loss": 0.5585312247276306, + "eval_mean_token_accuracy": 0.8575408472571262, + "eval_num_tokens": 4700145.0, + "eval_runtime": 79.6426, + "eval_samples_per_second": 17.265, + "eval_steps_per_second": 2.16, + "step": 2010 + }, + { + "entropy": 0.242041623765471, + "epoch": 5.099626400996264, + "grad_norm": 0.8845140337944031, + "learning_rate": 0.00010500581211723872, + "loss": 0.16415899276733398, + "mean_token_accuracy": 0.9458679586949975, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.22115826569497585, + "epoch": 5.224159402241594, + "grad_norm": 0.8571121692657471, + "learning_rate": 0.00010103602327710202, + "loss": 0.14506815910339355, + "mean_token_accuracy": 0.9509596079587936, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.2277975185215473, + "epoch": 5.348692403486924, + "grad_norm": 0.6461961269378662, + "learning_rate": 9.7049381353193e-05, + "loss": 0.15208842277526854, + "mean_token_accuracy": 0.9484275713562965, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.22508278906345366, + "epoch": 5.473225404732254, + "grad_norm": 0.6664603352546692, + "learning_rate": 9.305339983422608e-05, + "loss": 0.1507566452026367, + "mean_token_accuracy": 0.9494482669234275, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.22525668181478978, + "epoch": 5.597758405977584, + "grad_norm": 0.6928425431251526, + "learning_rate": 8.905560981093345e-05, + "loss": 0.15167025566101075, + "mean_token_accuracy": 0.947735041975975, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.21739320732653142, + "epoch": 5.722291407222914, + "grad_norm": 0.7611718773841858, + "learning_rate": 8.506354578247393e-05, + "loss": 0.1465705680847168, + "mean_token_accuracy": 0.9501141354441642, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.2239777410775423, + "epoch": 5.846824408468244, + "grad_norm": 0.7929556965827942, + "learning_rate": 8.108473145641841e-05, + "loss": 0.14866115570068358, + "mean_token_accuracy": 0.9497014331817627, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.21606657084077596, + "epoch": 5.971357409713574, + "grad_norm": 0.7851046323776245, + "learning_rate": 7.712666556907332e-05, + "loss": 0.14777716636657715, + "mean_token_accuracy": 0.9501199626922607, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.32871103122137313, + "eval_loss": 0.635863184928894, + "eval_mean_token_accuracy": 0.8554029128579206, + "eval_num_tokens": 5640174.0, + "eval_runtime": 79.4073, + "eval_samples_per_second": 17.316, + "eval_steps_per_second": 2.166, + "step": 2412 + }, + { + "entropy": 0.18288949677588964, + "epoch": 6.094645080946451, + "grad_norm": 0.39350658655166626, + "learning_rate": 7.319680775286681e-05, + "loss": 0.11040553092956543, + "mean_token_accuracy": 0.9622839373169523, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.16421406015753745, + "epoch": 6.219178082191781, + "grad_norm": 0.5661713480949402, + "learning_rate": 6.930256447743308e-05, + "loss": 0.0962723159790039, + "mean_token_accuracy": 0.967263371348381, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.16072349682450293, + "epoch": 6.34371108343711, + "grad_norm": 0.755388617515564, + "learning_rate": 6.545127509088995e-05, + "loss": 0.09449175834655761, + "mean_token_accuracy": 0.9674956914782524, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.15834762785583734, + "epoch": 6.468244084682441, + "grad_norm": 0.5315730571746826, + "learning_rate": 6.1650197987619e-05, + "loss": 0.09194418907165527, + "mean_token_accuracy": 0.9688713988661766, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.16119305711239576, + "epoch": 6.592777085927771, + "grad_norm": 0.6552009582519531, + "learning_rate": 5.7906496928615814e-05, + "loss": 0.0957893180847168, + "mean_token_accuracy": 0.9680398201942444, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.16668029114603997, + "epoch": 6.717310087173101, + "grad_norm": 0.6891521215438843, + "learning_rate": 5.4227227540192906e-05, + "loss": 0.09778289794921875, + "mean_token_accuracy": 0.9666138800978661, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.1648960181325674, + "epoch": 6.841843088418431, + "grad_norm": 0.5747121572494507, + "learning_rate": 5.061932401648035e-05, + "loss": 0.09770219802856445, + "mean_token_accuracy": 0.9660053381323814, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.15946010805666447, + "epoch": 6.966376089663761, + "grad_norm": 0.6286085247993469, + "learning_rate": 4.7089586050785194e-05, + "loss": 0.09402350425720214, + "mean_token_accuracy": 0.9677979084849357, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2758399397134781, + "eval_loss": 0.7382693290710449, + "eval_mean_token_accuracy": 0.8535292623348014, + "eval_num_tokens": 6580203.0, + "eval_runtime": 79.5655, + "eval_samples_per_second": 17.281, + "eval_steps_per_second": 2.162, + "step": 2814 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.713509679512576e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd48592ebfde6ffa9e1fa29412808c550e6c8898 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05833860659003597, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcf7771889782c86b574c7285983e2c77084ba1 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3216/trainer_state.json @@ -0,0 +1,762 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3216, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + }, + { + "entropy": 0.5661561889479859, + "epoch": 1.1195516811955168, + "grad_norm": 0.708380937576294, + "learning_rate": 0.00018411093929332744, + "loss": 0.4969264221191406, + "mean_token_accuracy": 0.8550105146085373, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.5690013617277145, + "epoch": 1.244084682440847, + "grad_norm": 0.8610317707061768, + "learning_rate": 0.00018386114730685091, + "loss": 0.49536712646484377, + "mean_token_accuracy": 0.8549688473343849, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.5661576768755913, + "epoch": 1.3686176836861768, + "grad_norm": 0.8410939574241638, + "learning_rate": 0.00018343840453669726, + "loss": 0.4957312774658203, + "mean_token_accuracy": 0.8546139162778854, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.5658727620542049, + "epoch": 1.4931506849315068, + "grad_norm": 0.6218363046646118, + "learning_rate": 0.00018284350771181608, + "loss": 0.4974911499023438, + "mean_token_accuracy": 0.855501549243927, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.5707372877001763, + "epoch": 1.6176836861768369, + "grad_norm": 0.8585225343704224, + "learning_rate": 0.0001820775780140605, + "loss": 0.49875095367431643, + "mean_token_accuracy": 0.8563189420104027, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5534089787304401, + "epoch": 1.7422166874221667, + "grad_norm": 0.6285793781280518, + "learning_rate": 0.00018114205896513374, + "loss": 0.48918685913085935, + "mean_token_accuracy": 0.8578527864813804, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5458812513947486, + "epoch": 1.8667496886674968, + "grad_norm": 0.71883225440979, + "learning_rate": 0.00018003871370603276, + "loss": 0.482542839050293, + "mean_token_accuracy": 0.8564121940732002, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5502609372138977, + "epoch": 1.9912826899128269, + "grad_norm": 0.6442635655403137, + "learning_rate": 0.00017876962167411625, + "loss": 0.48193965911865233, + "mean_token_accuracy": 0.8567470651865006, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5426353398104047, + "eval_loss": 0.5083975791931152, + "eval_mean_token_accuracy": 0.8554168769786524, + "eval_num_tokens": 1880058.0, + "eval_runtime": 79.8868, + "eval_samples_per_second": 17.212, + "eval_steps_per_second": 2.153, + "step": 804 + }, + { + "entropy": 0.4734916226430373, + "epoch": 2.1145703611457036, + "grad_norm": 0.7688794732093811, + "learning_rate": 0.00017733717468405966, + "loss": 0.40314048767089844, + "mean_token_accuracy": 0.8765652631268357, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.46802927538752553, + "epoch": 2.2391033623910337, + "grad_norm": 0.6301783919334412, + "learning_rate": 0.00017574407242008323, + "loss": 0.3979022979736328, + "mean_token_accuracy": 0.8784176242351532, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.4625655211508274, + "epoch": 2.3636363636363638, + "grad_norm": 0.5972099900245667, + "learning_rate": 0.00017399331734794884, + "loss": 0.4014426803588867, + "mean_token_accuracy": 0.8762527570128441, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.470507246106863, + "epoch": 2.488169364881694, + "grad_norm": 0.8060091733932495, + "learning_rate": 0.00017208820905631456, + "loss": 0.4042459869384766, + "mean_token_accuracy": 0.8747306689620018, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.4665305399894714, + "epoch": 2.6127023661270234, + "grad_norm": 0.6650318503379822, + "learning_rate": 0.0001700323380381121, + "loss": 0.40335521697998045, + "mean_token_accuracy": 0.8762387800216674, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.46196050629019736, + "epoch": 2.7372353673723535, + "grad_norm": 0.8204917907714844, + "learning_rate": 0.00016782957892366653, + "loss": 0.4012960052490234, + "mean_token_accuracy": 0.8761517608165741, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.4787097446620464, + "epoch": 2.8617683686176836, + "grad_norm": 0.6445292830467224, + "learning_rate": 0.00016548408317831203, + "loss": 0.4159215545654297, + "mean_token_accuracy": 0.8722453466057778, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.4758198526501656, + "epoch": 2.9863013698630136, + "grad_norm": 0.7560163736343384, + "learning_rate": 0.0001630002712782661, + "loss": 0.40826087951660156, + "mean_token_accuracy": 0.873089083135128, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.49463067810202754, + "eval_loss": 0.5086550116539001, + "eval_mean_token_accuracy": 0.857228794416716, + "eval_num_tokens": 2820087.0, + "eval_runtime": 79.7846, + "eval_samples_per_second": 17.234, + "eval_steps_per_second": 2.156, + "step": 1206 + }, + { + "entropy": 0.3859369834565153, + "epoch": 3.1095890410958904, + "grad_norm": 0.741679847240448, + "learning_rate": 0.00016038282437950814, + "loss": 0.32163028717041015, + "mean_token_accuracy": 0.8972690379378772, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.3825602190196514, + "epoch": 3.2341220423412205, + "grad_norm": 0.8756065368652344, + "learning_rate": 0.00015763667549536354, + "loss": 0.316363639831543, + "mean_token_accuracy": 0.8986149084568024, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.38129442662000657, + "epoch": 3.3586550435865505, + "grad_norm": 0.8529497981071472, + "learning_rate": 0.00015476700019942104, + "loss": 0.3164650917053223, + "mean_token_accuracy": 0.8971891248226166, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.3877453975379467, + "epoch": 3.4831880448318806, + "grad_norm": 0.7949019074440002, + "learning_rate": 0.0001517792068713048, + "loss": 0.31930156707763674, + "mean_token_accuracy": 0.8973806884884834, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.38809079021215437, + "epoch": 3.6077210460772102, + "grad_norm": 0.8050806522369385, + "learning_rate": 0.0001486789265036849, + "loss": 0.31977331161499023, + "mean_token_accuracy": 0.8985486030578613, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.3984291091561317, + "epoch": 3.7322540473225407, + "grad_norm": 0.7165534496307373, + "learning_rate": 0.00014547200208973657, + "loss": 0.32611679077148437, + "mean_token_accuracy": 0.895875141620636, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.39771562442183495, + "epoch": 3.8567870485678704, + "grad_norm": 0.6194931864738464, + "learning_rate": 0.00014216447761104936, + "loss": 0.32970951080322264, + "mean_token_accuracy": 0.8950823175907135, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.3997128288447857, + "epoch": 3.9813200498132004, + "grad_norm": 0.7835033535957336, + "learning_rate": 0.00013876258664674012, + "loss": 0.3263474655151367, + "mean_token_accuracy": 0.894331750869751, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45059218794800515, + "eval_loss": 0.5254963636398315, + "eval_mean_token_accuracy": 0.857301191535107, + "eval_num_tokens": 3760116.0, + "eval_runtime": 79.3533, + "eval_samples_per_second": 17.328, + "eval_steps_per_second": 2.168, + "step": 1608 + }, + { + "entropy": 0.31803849610415375, + "epoch": 4.104607721046078, + "grad_norm": 0.9932603240013123, + "learning_rate": 0.00013527274062523803, + "loss": 0.24486299514770507, + "mean_token_accuracy": 0.9204365689345081, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.30566568687558177, + "epoch": 4.229140722291407, + "grad_norm": 0.8069589138031006, + "learning_rate": 0.0001317015167408831, + "loss": 0.22590700149536133, + "mean_token_accuracy": 0.9249501618742942, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.2980187537521124, + "epoch": 4.353673723536737, + "grad_norm": 0.6587982773780823, + "learning_rate": 0.00012805564555811122, + "loss": 0.22620832443237304, + "mean_token_accuracy": 0.9248810124397278, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3055800000578165, + "epoch": 4.478206724782067, + "grad_norm": 0.8378816843032837, + "learning_rate": 0.00012434199832658783, + "loss": 0.23171533584594728, + "mean_token_accuracy": 0.9231083118915557, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.31550996117293834, + "epoch": 4.602739726027397, + "grad_norm": 0.749613881111145, + "learning_rate": 0.00012056757403119691, + "loss": 0.23747362136840822, + "mean_token_accuracy": 0.9206199425458909, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.30942291468381883, + "epoch": 4.7272727272727275, + "grad_norm": 0.7933925986289978, + "learning_rate": 0.0001167394862012918, + "loss": 0.23256269454956055, + "mean_token_accuracy": 0.9226748991012573, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.3098080876469612, + "epoch": 4.851805728518057, + "grad_norm": 0.8441582918167114, + "learning_rate": 0.00011286494950406773, + "loss": 0.23220256805419923, + "mean_token_accuracy": 0.9222584775090218, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3098256988823414, + "epoch": 4.976338729763388, + "grad_norm": 0.8682230114936829, + "learning_rate": 0.00010895126614732359, + "loss": 0.23128936767578126, + "mean_token_accuracy": 0.923048702776432, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.39352814238085304, + "eval_loss": 0.5585312247276306, + "eval_mean_token_accuracy": 0.8575408472571262, + "eval_num_tokens": 4700145.0, + "eval_runtime": 79.6426, + "eval_samples_per_second": 17.265, + "eval_steps_per_second": 2.16, + "step": 2010 + }, + { + "entropy": 0.242041623765471, + "epoch": 5.099626400996264, + "grad_norm": 0.8845140337944031, + "learning_rate": 0.00010500581211723872, + "loss": 0.16415899276733398, + "mean_token_accuracy": 0.9458679586949975, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.22115826569497585, + "epoch": 5.224159402241594, + "grad_norm": 0.8571121692657471, + "learning_rate": 0.00010103602327710202, + "loss": 0.14506815910339355, + "mean_token_accuracy": 0.9509596079587936, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.2277975185215473, + "epoch": 5.348692403486924, + "grad_norm": 0.6461961269378662, + "learning_rate": 9.7049381353193e-05, + "loss": 0.15208842277526854, + "mean_token_accuracy": 0.9484275713562965, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.22508278906345366, + "epoch": 5.473225404732254, + "grad_norm": 0.6664603352546692, + "learning_rate": 9.305339983422608e-05, + "loss": 0.1507566452026367, + "mean_token_accuracy": 0.9494482669234275, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.22525668181478978, + "epoch": 5.597758405977584, + "grad_norm": 0.6928425431251526, + "learning_rate": 8.905560981093345e-05, + "loss": 0.15167025566101075, + "mean_token_accuracy": 0.947735041975975, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.21739320732653142, + "epoch": 5.722291407222914, + "grad_norm": 0.7611718773841858, + "learning_rate": 8.506354578247393e-05, + "loss": 0.1465705680847168, + "mean_token_accuracy": 0.9501141354441642, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.2239777410775423, + "epoch": 5.846824408468244, + "grad_norm": 0.7929556965827942, + "learning_rate": 8.108473145641841e-05, + "loss": 0.14866115570068358, + "mean_token_accuracy": 0.9497014331817627, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.21606657084077596, + "epoch": 5.971357409713574, + "grad_norm": 0.7851046323776245, + "learning_rate": 7.712666556907332e-05, + "loss": 0.14777716636657715, + "mean_token_accuracy": 0.9501199626922607, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.32871103122137313, + "eval_loss": 0.635863184928894, + "eval_mean_token_accuracy": 0.8554029128579206, + "eval_num_tokens": 5640174.0, + "eval_runtime": 79.4073, + "eval_samples_per_second": 17.316, + "eval_steps_per_second": 2.166, + "step": 2412 + }, + { + "entropy": 0.18288949677588964, + "epoch": 6.094645080946451, + "grad_norm": 0.39350658655166626, + "learning_rate": 7.319680775286681e-05, + "loss": 0.11040553092956543, + "mean_token_accuracy": 0.9622839373169523, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.16421406015753745, + "epoch": 6.219178082191781, + "grad_norm": 0.5661713480949402, + "learning_rate": 6.930256447743308e-05, + "loss": 0.0962723159790039, + "mean_token_accuracy": 0.967263371348381, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.16072349682450293, + "epoch": 6.34371108343711, + "grad_norm": 0.755388617515564, + "learning_rate": 6.545127509088995e-05, + "loss": 0.09449175834655761, + "mean_token_accuracy": 0.9674956914782524, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.15834762785583734, + "epoch": 6.468244084682441, + "grad_norm": 0.5315730571746826, + "learning_rate": 6.1650197987619e-05, + "loss": 0.09194418907165527, + "mean_token_accuracy": 0.9688713988661766, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.16119305711239576, + "epoch": 6.592777085927771, + "grad_norm": 0.6552009582519531, + "learning_rate": 5.7906496928615814e-05, + "loss": 0.0957893180847168, + "mean_token_accuracy": 0.9680398201942444, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.16668029114603997, + "epoch": 6.717310087173101, + "grad_norm": 0.6891521215438843, + "learning_rate": 5.4227227540192906e-05, + "loss": 0.09778289794921875, + "mean_token_accuracy": 0.9666138800978661, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.1648960181325674, + "epoch": 6.841843088418431, + "grad_norm": 0.5747121572494507, + "learning_rate": 5.061932401648035e-05, + "loss": 0.09770219802856445, + "mean_token_accuracy": 0.9660053381323814, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.15946010805666447, + "epoch": 6.966376089663761, + "grad_norm": 0.6286085247993469, + "learning_rate": 4.7089586050785194e-05, + "loss": 0.09402350425720214, + "mean_token_accuracy": 0.9677979084849357, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2758399397134781, + "eval_loss": 0.7382693290710449, + "eval_mean_token_accuracy": 0.8535292623348014, + "eval_num_tokens": 6580203.0, + "eval_runtime": 79.5655, + "eval_samples_per_second": 17.281, + "eval_steps_per_second": 2.162, + "step": 2814 + }, + { + "entropy": 0.15347467818193966, + "epoch": 7.089663760896638, + "grad_norm": 0.45861202478408813, + "learning_rate": 4.36446660204407e-05, + "loss": 0.08010212898254394, + "mean_token_accuracy": 0.9720996783839332, + "num_tokens": 6661238.0, + "step": 2850 + }, + { + "entropy": 0.1345258642360568, + "epoch": 7.214196762141968, + "grad_norm": 0.4790540933609009, + "learning_rate": 4.029105644929618e-05, + "loss": 0.07323225498199464, + "mean_token_accuracy": 0.9749647602438927, + "num_tokens": 6774391.0, + "step": 2900 + }, + { + "entropy": 0.13495642982423306, + "epoch": 7.338729763387297, + "grad_norm": 0.41177162528038025, + "learning_rate": 3.7035077771478046e-05, + "loss": 0.0723848009109497, + "mean_token_accuracy": 0.9750138682126999, + "num_tokens": 6892967.0, + "step": 2950 + }, + { + "entropy": 0.14119064182043076, + "epoch": 7.463262764632628, + "grad_norm": 0.3276011049747467, + "learning_rate": 3.3882866419481806e-05, + "loss": 0.07503739356994629, + "mean_token_accuracy": 0.973833509683609, + "num_tokens": 7006692.0, + "step": 3000 + }, + { + "entropy": 0.13939726265147329, + "epoch": 7.587795765877957, + "grad_norm": 0.29367685317993164, + "learning_rate": 3.084036325904632e-05, + "loss": 0.07321533203125, + "mean_token_accuracy": 0.9747413221001625, + "num_tokens": 7124768.0, + "step": 3050 + }, + { + "entropy": 0.1330118723027408, + "epoch": 7.712328767123288, + "grad_norm": 0.26096275448799133, + "learning_rate": 2.7913302392605573e-05, + "loss": 0.0725630760192871, + "mean_token_accuracy": 0.9755767813324928, + "num_tokens": 7242764.0, + "step": 3100 + }, + { + "entropy": 0.12999726412817836, + "epoch": 7.8368617683686175, + "grad_norm": 0.4909002482891083, + "learning_rate": 2.5107200352420576e-05, + "loss": 0.06970284938812256, + "mean_token_accuracy": 0.9758264967799186, + "num_tokens": 7366604.0, + "step": 3150 + }, + { + "entropy": 0.13837616711854936, + "epoch": 7.961394769613948, + "grad_norm": 0.37932804226875305, + "learning_rate": 2.2427345703758274e-05, + "loss": 0.07299811363220216, + "mean_token_accuracy": 0.9738594210147857, + "num_tokens": 7481687.0, + "step": 3200 + }, + { + "epoch": 8.0, + "eval_entropy": 0.25052128273040747, + "eval_loss": 0.8225418925285339, + "eval_mean_token_accuracy": 0.8537235357040582, + "eval_num_tokens": 7520232.0, + "eval_runtime": 79.595, + "eval_samples_per_second": 17.275, + "eval_steps_per_second": 2.161, + "step": 3216 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.103242744371712e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd48592ebfde6ffa9e1fa29412808c550e6c8898 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05833860659003597, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6d9db74648e03892f94c3a6c8e5f5ded64ec5017 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3618/trainer_state.json @@ -0,0 +1,853 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3618, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + }, + { + "entropy": 0.5661561889479859, + "epoch": 1.1195516811955168, + "grad_norm": 0.708380937576294, + "learning_rate": 0.00018411093929332744, + "loss": 0.4969264221191406, + "mean_token_accuracy": 0.8550105146085373, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.5690013617277145, + "epoch": 1.244084682440847, + "grad_norm": 0.8610317707061768, + "learning_rate": 0.00018386114730685091, + "loss": 0.49536712646484377, + "mean_token_accuracy": 0.8549688473343849, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.5661576768755913, + "epoch": 1.3686176836861768, + "grad_norm": 0.8410939574241638, + "learning_rate": 0.00018343840453669726, + "loss": 0.4957312774658203, + "mean_token_accuracy": 0.8546139162778854, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.5658727620542049, + "epoch": 1.4931506849315068, + "grad_norm": 0.6218363046646118, + "learning_rate": 0.00018284350771181608, + "loss": 0.4974911499023438, + "mean_token_accuracy": 0.855501549243927, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.5707372877001763, + "epoch": 1.6176836861768369, + "grad_norm": 0.8585225343704224, + "learning_rate": 0.0001820775780140605, + "loss": 0.49875095367431643, + "mean_token_accuracy": 0.8563189420104027, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5534089787304401, + "epoch": 1.7422166874221667, + "grad_norm": 0.6285793781280518, + "learning_rate": 0.00018114205896513374, + "loss": 0.48918685913085935, + "mean_token_accuracy": 0.8578527864813804, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5458812513947486, + "epoch": 1.8667496886674968, + "grad_norm": 0.71883225440979, + "learning_rate": 0.00018003871370603276, + "loss": 0.482542839050293, + "mean_token_accuracy": 0.8564121940732002, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5502609372138977, + "epoch": 1.9912826899128269, + "grad_norm": 0.6442635655403137, + "learning_rate": 0.00017876962167411625, + "loss": 0.48193965911865233, + "mean_token_accuracy": 0.8567470651865006, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5426353398104047, + "eval_loss": 0.5083975791931152, + "eval_mean_token_accuracy": 0.8554168769786524, + "eval_num_tokens": 1880058.0, + "eval_runtime": 79.8868, + "eval_samples_per_second": 17.212, + "eval_steps_per_second": 2.153, + "step": 804 + }, + { + "entropy": 0.4734916226430373, + "epoch": 2.1145703611457036, + "grad_norm": 0.7688794732093811, + "learning_rate": 0.00017733717468405966, + "loss": 0.40314048767089844, + "mean_token_accuracy": 0.8765652631268357, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.46802927538752553, + "epoch": 2.2391033623910337, + "grad_norm": 0.6301783919334412, + "learning_rate": 0.00017574407242008323, + "loss": 0.3979022979736328, + "mean_token_accuracy": 0.8784176242351532, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.4625655211508274, + "epoch": 2.3636363636363638, + "grad_norm": 0.5972099900245667, + "learning_rate": 0.00017399331734794884, + "loss": 0.4014426803588867, + "mean_token_accuracy": 0.8762527570128441, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.470507246106863, + "epoch": 2.488169364881694, + "grad_norm": 0.8060091733932495, + "learning_rate": 0.00017208820905631456, + "loss": 0.4042459869384766, + "mean_token_accuracy": 0.8747306689620018, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.4665305399894714, + "epoch": 2.6127023661270234, + "grad_norm": 0.6650318503379822, + "learning_rate": 0.0001700323380381121, + "loss": 0.40335521697998045, + "mean_token_accuracy": 0.8762387800216674, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.46196050629019736, + "epoch": 2.7372353673723535, + "grad_norm": 0.8204917907714844, + "learning_rate": 0.00016782957892366653, + "loss": 0.4012960052490234, + "mean_token_accuracy": 0.8761517608165741, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.4787097446620464, + "epoch": 2.8617683686176836, + "grad_norm": 0.6445292830467224, + "learning_rate": 0.00016548408317831203, + "loss": 0.4159215545654297, + "mean_token_accuracy": 0.8722453466057778, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.4758198526501656, + "epoch": 2.9863013698630136, + "grad_norm": 0.7560163736343384, + "learning_rate": 0.0001630002712782661, + "loss": 0.40826087951660156, + "mean_token_accuracy": 0.873089083135128, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.49463067810202754, + "eval_loss": 0.5086550116539001, + "eval_mean_token_accuracy": 0.857228794416716, + "eval_num_tokens": 2820087.0, + "eval_runtime": 79.7846, + "eval_samples_per_second": 17.234, + "eval_steps_per_second": 2.156, + "step": 1206 + }, + { + "entropy": 0.3859369834565153, + "epoch": 3.1095890410958904, + "grad_norm": 0.741679847240448, + "learning_rate": 0.00016038282437950814, + "loss": 0.32163028717041015, + "mean_token_accuracy": 0.8972690379378772, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.3825602190196514, + "epoch": 3.2341220423412205, + "grad_norm": 0.8756065368652344, + "learning_rate": 0.00015763667549536354, + "loss": 0.316363639831543, + "mean_token_accuracy": 0.8986149084568024, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.38129442662000657, + "epoch": 3.3586550435865505, + "grad_norm": 0.8529497981071472, + "learning_rate": 0.00015476700019942104, + "loss": 0.3164650917053223, + "mean_token_accuracy": 0.8971891248226166, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.3877453975379467, + "epoch": 3.4831880448318806, + "grad_norm": 0.7949019074440002, + "learning_rate": 0.0001517792068713048, + "loss": 0.31930156707763674, + "mean_token_accuracy": 0.8973806884884834, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.38809079021215437, + "epoch": 3.6077210460772102, + "grad_norm": 0.8050806522369385, + "learning_rate": 0.0001486789265036849, + "loss": 0.31977331161499023, + "mean_token_accuracy": 0.8985486030578613, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.3984291091561317, + "epoch": 3.7322540473225407, + "grad_norm": 0.7165534496307373, + "learning_rate": 0.00014547200208973657, + "loss": 0.32611679077148437, + "mean_token_accuracy": 0.895875141620636, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.39771562442183495, + "epoch": 3.8567870485678704, + "grad_norm": 0.6194931864738464, + "learning_rate": 0.00014216447761104936, + "loss": 0.32970951080322264, + "mean_token_accuracy": 0.8950823175907135, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.3997128288447857, + "epoch": 3.9813200498132004, + "grad_norm": 0.7835033535957336, + "learning_rate": 0.00013876258664674012, + "loss": 0.3263474655151367, + "mean_token_accuracy": 0.894331750869751, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45059218794800515, + "eval_loss": 0.5254963636398315, + "eval_mean_token_accuracy": 0.857301191535107, + "eval_num_tokens": 3760116.0, + "eval_runtime": 79.3533, + "eval_samples_per_second": 17.328, + "eval_steps_per_second": 2.168, + "step": 1608 + }, + { + "entropy": 0.31803849610415375, + "epoch": 4.104607721046078, + "grad_norm": 0.9932603240013123, + "learning_rate": 0.00013527274062523803, + "loss": 0.24486299514770507, + "mean_token_accuracy": 0.9204365689345081, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.30566568687558177, + "epoch": 4.229140722291407, + "grad_norm": 0.8069589138031006, + "learning_rate": 0.0001317015167408831, + "loss": 0.22590700149536133, + "mean_token_accuracy": 0.9249501618742942, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.2980187537521124, + "epoch": 4.353673723536737, + "grad_norm": 0.6587982773780823, + "learning_rate": 0.00012805564555811122, + "loss": 0.22620832443237304, + "mean_token_accuracy": 0.9248810124397278, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3055800000578165, + "epoch": 4.478206724782067, + "grad_norm": 0.8378816843032837, + "learning_rate": 0.00012434199832658783, + "loss": 0.23171533584594728, + "mean_token_accuracy": 0.9231083118915557, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.31550996117293834, + "epoch": 4.602739726027397, + "grad_norm": 0.749613881111145, + "learning_rate": 0.00012056757403119691, + "loss": 0.23747362136840822, + "mean_token_accuracy": 0.9206199425458909, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.30942291468381883, + "epoch": 4.7272727272727275, + "grad_norm": 0.7933925986289978, + "learning_rate": 0.0001167394862012918, + "loss": 0.23256269454956055, + "mean_token_accuracy": 0.9226748991012573, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.3098080876469612, + "epoch": 4.851805728518057, + "grad_norm": 0.8441582918167114, + "learning_rate": 0.00011286494950406773, + "loss": 0.23220256805419923, + "mean_token_accuracy": 0.9222584775090218, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3098256988823414, + "epoch": 4.976338729763388, + "grad_norm": 0.8682230114936829, + "learning_rate": 0.00010895126614732359, + "loss": 0.23128936767578126, + "mean_token_accuracy": 0.923048702776432, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.39352814238085304, + "eval_loss": 0.5585312247276306, + "eval_mean_token_accuracy": 0.8575408472571262, + "eval_num_tokens": 4700145.0, + "eval_runtime": 79.6426, + "eval_samples_per_second": 17.265, + "eval_steps_per_second": 2.16, + "step": 2010 + }, + { + "entropy": 0.242041623765471, + "epoch": 5.099626400996264, + "grad_norm": 0.8845140337944031, + "learning_rate": 0.00010500581211723872, + "loss": 0.16415899276733398, + "mean_token_accuracy": 0.9458679586949975, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.22115826569497585, + "epoch": 5.224159402241594, + "grad_norm": 0.8571121692657471, + "learning_rate": 0.00010103602327710202, + "loss": 0.14506815910339355, + "mean_token_accuracy": 0.9509596079587936, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.2277975185215473, + "epoch": 5.348692403486924, + "grad_norm": 0.6461961269378662, + "learning_rate": 9.7049381353193e-05, + "loss": 0.15208842277526854, + "mean_token_accuracy": 0.9484275713562965, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.22508278906345366, + "epoch": 5.473225404732254, + "grad_norm": 0.6664603352546692, + "learning_rate": 9.305339983422608e-05, + "loss": 0.1507566452026367, + "mean_token_accuracy": 0.9494482669234275, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.22525668181478978, + "epoch": 5.597758405977584, + "grad_norm": 0.6928425431251526, + "learning_rate": 8.905560981093345e-05, + "loss": 0.15167025566101075, + "mean_token_accuracy": 0.947735041975975, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.21739320732653142, + "epoch": 5.722291407222914, + "grad_norm": 0.7611718773841858, + "learning_rate": 8.506354578247393e-05, + "loss": 0.1465705680847168, + "mean_token_accuracy": 0.9501141354441642, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.2239777410775423, + "epoch": 5.846824408468244, + "grad_norm": 0.7929556965827942, + "learning_rate": 8.108473145641841e-05, + "loss": 0.14866115570068358, + "mean_token_accuracy": 0.9497014331817627, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.21606657084077596, + "epoch": 5.971357409713574, + "grad_norm": 0.7851046323776245, + "learning_rate": 7.712666556907332e-05, + "loss": 0.14777716636657715, + "mean_token_accuracy": 0.9501199626922607, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.32871103122137313, + "eval_loss": 0.635863184928894, + "eval_mean_token_accuracy": 0.8554029128579206, + "eval_num_tokens": 5640174.0, + "eval_runtime": 79.4073, + "eval_samples_per_second": 17.316, + "eval_steps_per_second": 2.166, + "step": 2412 + }, + { + "entropy": 0.18288949677588964, + "epoch": 6.094645080946451, + "grad_norm": 0.39350658655166626, + "learning_rate": 7.319680775286681e-05, + "loss": 0.11040553092956543, + "mean_token_accuracy": 0.9622839373169523, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.16421406015753745, + "epoch": 6.219178082191781, + "grad_norm": 0.5661713480949402, + "learning_rate": 6.930256447743308e-05, + "loss": 0.0962723159790039, + "mean_token_accuracy": 0.967263371348381, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.16072349682450293, + "epoch": 6.34371108343711, + "grad_norm": 0.755388617515564, + "learning_rate": 6.545127509088995e-05, + "loss": 0.09449175834655761, + "mean_token_accuracy": 0.9674956914782524, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.15834762785583734, + "epoch": 6.468244084682441, + "grad_norm": 0.5315730571746826, + "learning_rate": 6.1650197987619e-05, + "loss": 0.09194418907165527, + "mean_token_accuracy": 0.9688713988661766, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.16119305711239576, + "epoch": 6.592777085927771, + "grad_norm": 0.6552009582519531, + "learning_rate": 5.7906496928615814e-05, + "loss": 0.0957893180847168, + "mean_token_accuracy": 0.9680398201942444, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.16668029114603997, + "epoch": 6.717310087173101, + "grad_norm": 0.6891521215438843, + "learning_rate": 5.4227227540192906e-05, + "loss": 0.09778289794921875, + "mean_token_accuracy": 0.9666138800978661, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.1648960181325674, + "epoch": 6.841843088418431, + "grad_norm": 0.5747121572494507, + "learning_rate": 5.061932401648035e-05, + "loss": 0.09770219802856445, + "mean_token_accuracy": 0.9660053381323814, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.15946010805666447, + "epoch": 6.966376089663761, + "grad_norm": 0.6286085247993469, + "learning_rate": 4.7089586050785194e-05, + "loss": 0.09402350425720214, + "mean_token_accuracy": 0.9677979084849357, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2758399397134781, + "eval_loss": 0.7382693290710449, + "eval_mean_token_accuracy": 0.8535292623348014, + "eval_num_tokens": 6580203.0, + "eval_runtime": 79.5655, + "eval_samples_per_second": 17.281, + "eval_steps_per_second": 2.162, + "step": 2814 + }, + { + "entropy": 0.15347467818193966, + "epoch": 7.089663760896638, + "grad_norm": 0.45861202478408813, + "learning_rate": 4.36446660204407e-05, + "loss": 0.08010212898254394, + "mean_token_accuracy": 0.9720996783839332, + "num_tokens": 6661238.0, + "step": 2850 + }, + { + "entropy": 0.1345258642360568, + "epoch": 7.214196762141968, + "grad_norm": 0.4790540933609009, + "learning_rate": 4.029105644929618e-05, + "loss": 0.07323225498199464, + "mean_token_accuracy": 0.9749647602438927, + "num_tokens": 6774391.0, + "step": 2900 + }, + { + "entropy": 0.13495642982423306, + "epoch": 7.338729763387297, + "grad_norm": 0.41177162528038025, + "learning_rate": 3.7035077771478046e-05, + "loss": 0.0723848009109497, + "mean_token_accuracy": 0.9750138682126999, + "num_tokens": 6892967.0, + "step": 2950 + }, + { + "entropy": 0.14119064182043076, + "epoch": 7.463262764632628, + "grad_norm": 0.3276011049747467, + "learning_rate": 3.3882866419481806e-05, + "loss": 0.07503739356994629, + "mean_token_accuracy": 0.973833509683609, + "num_tokens": 7006692.0, + "step": 3000 + }, + { + "entropy": 0.13939726265147329, + "epoch": 7.587795765877957, + "grad_norm": 0.29367685317993164, + "learning_rate": 3.084036325904632e-05, + "loss": 0.07321533203125, + "mean_token_accuracy": 0.9747413221001625, + "num_tokens": 7124768.0, + "step": 3050 + }, + { + "entropy": 0.1330118723027408, + "epoch": 7.712328767123288, + "grad_norm": 0.26096275448799133, + "learning_rate": 2.7913302392605573e-05, + "loss": 0.0725630760192871, + "mean_token_accuracy": 0.9755767813324928, + "num_tokens": 7242764.0, + "step": 3100 + }, + { + "entropy": 0.12999726412817836, + "epoch": 7.8368617683686175, + "grad_norm": 0.4909002482891083, + "learning_rate": 2.5107200352420576e-05, + "loss": 0.06970284938812256, + "mean_token_accuracy": 0.9758264967799186, + "num_tokens": 7366604.0, + "step": 3150 + }, + { + "entropy": 0.13837616711854936, + "epoch": 7.961394769613948, + "grad_norm": 0.37932804226875305, + "learning_rate": 2.2427345703758274e-05, + "loss": 0.07299811363220216, + "mean_token_accuracy": 0.9738594210147857, + "num_tokens": 7481687.0, + "step": 3200 + }, + { + "epoch": 8.0, + "eval_entropy": 0.25052128273040747, + "eval_loss": 0.8225418925285339, + "eval_mean_token_accuracy": 0.8537235357040582, + "eval_num_tokens": 7520232.0, + "eval_runtime": 79.595, + "eval_samples_per_second": 17.275, + "eval_steps_per_second": 2.161, + "step": 3216 + }, + { + "entropy": 0.12771825577047738, + "epoch": 8.084682440846825, + "grad_norm": 0.1904180347919464, + "learning_rate": 1.9878789077711986e-05, + "loss": 0.06388124942779541, + "mean_token_accuracy": 0.9773780998557505, + "num_tokens": 7601281.0, + "step": 3250 + }, + { + "entropy": 0.12670317662879824, + "epoch": 8.209215442092155, + "grad_norm": 0.31729868054389954, + "learning_rate": 1.7466333652448554e-05, + "loss": 0.06243476390838623, + "mean_token_accuracy": 0.977044507265091, + "num_tokens": 7720929.0, + "step": 3300 + }, + { + "entropy": 0.12671989548951387, + "epoch": 8.333748443337484, + "grad_norm": 0.2833121120929718, + "learning_rate": 1.5194526100821374e-05, + "loss": 0.06502001285552979, + "mean_token_accuracy": 0.9766163811087608, + "num_tokens": 7834637.0, + "step": 3350 + }, + { + "entropy": 0.12553330410271882, + "epoch": 8.458281444582815, + "grad_norm": 0.3116062581539154, + "learning_rate": 1.3067648021410455e-05, + "loss": 0.06511738300323486, + "mean_token_accuracy": 0.9765481826663017, + "num_tokens": 7947969.0, + "step": 3400 + }, + { + "entropy": 0.12742114875465632, + "epoch": 8.582814445828145, + "grad_norm": 0.18022380769252777, + "learning_rate": 1.1089707869138902e-05, + "loss": 0.06584225654602051, + "mean_token_accuracy": 0.9765808299183846, + "num_tokens": 8063392.0, + "step": 3450 + }, + { + "entropy": 0.1201029148697853, + "epoch": 8.707347447073474, + "grad_norm": 0.20968234539031982, + "learning_rate": 9.26443340067398e-06, + "loss": 0.06199615478515625, + "mean_token_accuracy": 0.9771515518426895, + "num_tokens": 8186140.0, + "step": 3500 + }, + { + "entropy": 0.1267964872159064, + "epoch": 8.831880448318804, + "grad_norm": 0.29122787714004517, + "learning_rate": 7.5952646488506875e-06, + "loss": 0.0641715908050537, + "mean_token_accuracy": 0.9764416518807412, + "num_tokens": 8304658.0, + "step": 3550 + }, + { + "entropy": 0.13317630462348462, + "epoch": 8.956413449564135, + "grad_norm": 0.2513009309768677, + "learning_rate": 6.0853474393585395e-06, + "loss": 0.06596921443939209, + "mean_token_accuracy": 0.9751221585273743, + "num_tokens": 8419672.0, + "step": 3600 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2314538847395154, + "eval_loss": 0.9136112332344055, + "eval_mean_token_accuracy": 0.8535278571206469, + "eval_num_tokens": 8460261.0, + "eval_runtime": 79.6191, + "eval_samples_per_second": 17.27, + "eval_steps_per_second": 2.16, + "step": 3618 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.4901950953099264e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd48592ebfde6ffa9e1fa29412808c550e6c8898 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05833860659003597, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b239204fcd5744d9d77852fddfb15d06f5bb718a --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-402/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 402, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.894650812304794e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd48592ebfde6ffa9e1fa29412808c550e6c8898 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05833860659003597, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ad195cfc2573b33f98980cfaf3d2451c362d9f41 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4020/trainer_state.json @@ -0,0 +1,944 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4020, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + }, + { + "entropy": 0.5661561889479859, + "epoch": 1.1195516811955168, + "grad_norm": 0.708380937576294, + "learning_rate": 0.00018411093929332744, + "loss": 0.4969264221191406, + "mean_token_accuracy": 0.8550105146085373, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.5690013617277145, + "epoch": 1.244084682440847, + "grad_norm": 0.8610317707061768, + "learning_rate": 0.00018386114730685091, + "loss": 0.49536712646484377, + "mean_token_accuracy": 0.8549688473343849, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.5661576768755913, + "epoch": 1.3686176836861768, + "grad_norm": 0.8410939574241638, + "learning_rate": 0.00018343840453669726, + "loss": 0.4957312774658203, + "mean_token_accuracy": 0.8546139162778854, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.5658727620542049, + "epoch": 1.4931506849315068, + "grad_norm": 0.6218363046646118, + "learning_rate": 0.00018284350771181608, + "loss": 0.4974911499023438, + "mean_token_accuracy": 0.855501549243927, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.5707372877001763, + "epoch": 1.6176836861768369, + "grad_norm": 0.8585225343704224, + "learning_rate": 0.0001820775780140605, + "loss": 0.49875095367431643, + "mean_token_accuracy": 0.8563189420104027, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5534089787304401, + "epoch": 1.7422166874221667, + "grad_norm": 0.6285793781280518, + "learning_rate": 0.00018114205896513374, + "loss": 0.48918685913085935, + "mean_token_accuracy": 0.8578527864813804, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5458812513947486, + "epoch": 1.8667496886674968, + "grad_norm": 0.71883225440979, + "learning_rate": 0.00018003871370603276, + "loss": 0.482542839050293, + "mean_token_accuracy": 0.8564121940732002, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5502609372138977, + "epoch": 1.9912826899128269, + "grad_norm": 0.6442635655403137, + "learning_rate": 0.00017876962167411625, + "loss": 0.48193965911865233, + "mean_token_accuracy": 0.8567470651865006, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5426353398104047, + "eval_loss": 0.5083975791931152, + "eval_mean_token_accuracy": 0.8554168769786524, + "eval_num_tokens": 1880058.0, + "eval_runtime": 79.8868, + "eval_samples_per_second": 17.212, + "eval_steps_per_second": 2.153, + "step": 804 + }, + { + "entropy": 0.4734916226430373, + "epoch": 2.1145703611457036, + "grad_norm": 0.7688794732093811, + "learning_rate": 0.00017733717468405966, + "loss": 0.40314048767089844, + "mean_token_accuracy": 0.8765652631268357, + "num_tokens": 1989183.0, + "step": 850 + }, + { + "entropy": 0.46802927538752553, + "epoch": 2.2391033623910337, + "grad_norm": 0.6301783919334412, + "learning_rate": 0.00017574407242008323, + "loss": 0.3979022979736328, + "mean_token_accuracy": 0.8784176242351532, + "num_tokens": 2106590.0, + "step": 900 + }, + { + "entropy": 0.4625655211508274, + "epoch": 2.3636363636363638, + "grad_norm": 0.5972099900245667, + "learning_rate": 0.00017399331734794884, + "loss": 0.4014426803588867, + "mean_token_accuracy": 0.8762527570128441, + "num_tokens": 2218296.0, + "step": 950 + }, + { + "entropy": 0.470507246106863, + "epoch": 2.488169364881694, + "grad_norm": 0.8060091733932495, + "learning_rate": 0.00017208820905631456, + "loss": 0.4042459869384766, + "mean_token_accuracy": 0.8747306689620018, + "num_tokens": 2336965.0, + "step": 1000 + }, + { + "entropy": 0.4665305399894714, + "epoch": 2.6127023661270234, + "grad_norm": 0.6650318503379822, + "learning_rate": 0.0001700323380381121, + "loss": 0.40335521697998045, + "mean_token_accuracy": 0.8762387800216674, + "num_tokens": 2454272.0, + "step": 1050 + }, + { + "entropy": 0.46196050629019736, + "epoch": 2.7372353673723535, + "grad_norm": 0.8204917907714844, + "learning_rate": 0.00016782957892366653, + "loss": 0.4012960052490234, + "mean_token_accuracy": 0.8761517608165741, + "num_tokens": 2576164.0, + "step": 1100 + }, + { + "entropy": 0.4787097446620464, + "epoch": 2.8617683686176836, + "grad_norm": 0.6445292830467224, + "learning_rate": 0.00016548408317831203, + "loss": 0.4159215545654297, + "mean_token_accuracy": 0.8722453466057778, + "num_tokens": 2690325.0, + "step": 1150 + }, + { + "entropy": 0.4758198526501656, + "epoch": 2.9863013698630136, + "grad_norm": 0.7560163736343384, + "learning_rate": 0.0001630002712782661, + "loss": 0.40826087951660156, + "mean_token_accuracy": 0.873089083135128, + "num_tokens": 2807305.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.49463067810202754, + "eval_loss": 0.5086550116539001, + "eval_mean_token_accuracy": 0.857228794416716, + "eval_num_tokens": 2820087.0, + "eval_runtime": 79.7846, + "eval_samples_per_second": 17.234, + "eval_steps_per_second": 2.156, + "step": 1206 + }, + { + "entropy": 0.3859369834565153, + "epoch": 3.1095890410958904, + "grad_norm": 0.741679847240448, + "learning_rate": 0.00016038282437950814, + "loss": 0.32163028717041015, + "mean_token_accuracy": 0.8972690379378772, + "num_tokens": 2925625.0, + "step": 1250 + }, + { + "entropy": 0.3825602190196514, + "epoch": 3.2341220423412205, + "grad_norm": 0.8756065368652344, + "learning_rate": 0.00015763667549536354, + "loss": 0.316363639831543, + "mean_token_accuracy": 0.8986149084568024, + "num_tokens": 3041486.0, + "step": 1300 + }, + { + "entropy": 0.38129442662000657, + "epoch": 3.3586550435865505, + "grad_norm": 0.8529497981071472, + "learning_rate": 0.00015476700019942104, + "loss": 0.3164650917053223, + "mean_token_accuracy": 0.8971891248226166, + "num_tokens": 3160915.0, + "step": 1350 + }, + { + "entropy": 0.3877453975379467, + "epoch": 3.4831880448318806, + "grad_norm": 0.7949019074440002, + "learning_rate": 0.0001517792068713048, + "loss": 0.31930156707763674, + "mean_token_accuracy": 0.8973806884884834, + "num_tokens": 3278772.0, + "step": 1400 + }, + { + "entropy": 0.38809079021215437, + "epoch": 3.6077210460772102, + "grad_norm": 0.8050806522369385, + "learning_rate": 0.0001486789265036849, + "loss": 0.31977331161499023, + "mean_token_accuracy": 0.8985486030578613, + "num_tokens": 3395664.0, + "step": 1450 + }, + { + "entropy": 0.3984291091561317, + "epoch": 3.7322540473225407, + "grad_norm": 0.7165534496307373, + "learning_rate": 0.00014547200208973657, + "loss": 0.32611679077148437, + "mean_token_accuracy": 0.895875141620636, + "num_tokens": 3512643.0, + "step": 1500 + }, + { + "entropy": 0.39771562442183495, + "epoch": 3.8567870485678704, + "grad_norm": 0.6194931864738464, + "learning_rate": 0.00014216447761104936, + "loss": 0.32970951080322264, + "mean_token_accuracy": 0.8950823175907135, + "num_tokens": 3628481.0, + "step": 1550 + }, + { + "entropy": 0.3997128288447857, + "epoch": 3.9813200498132004, + "grad_norm": 0.7835033535957336, + "learning_rate": 0.00013876258664674012, + "loss": 0.3263474655151367, + "mean_token_accuracy": 0.894331750869751, + "num_tokens": 3741971.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45059218794800515, + "eval_loss": 0.5254963636398315, + "eval_mean_token_accuracy": 0.857301191535107, + "eval_num_tokens": 3760116.0, + "eval_runtime": 79.3533, + "eval_samples_per_second": 17.328, + "eval_steps_per_second": 2.168, + "step": 1608 + }, + { + "entropy": 0.31803849610415375, + "epoch": 4.104607721046078, + "grad_norm": 0.9932603240013123, + "learning_rate": 0.00013527274062523803, + "loss": 0.24486299514770507, + "mean_token_accuracy": 0.9204365689345081, + "num_tokens": 3857514.0, + "step": 1650 + }, + { + "entropy": 0.30566568687558177, + "epoch": 4.229140722291407, + "grad_norm": 0.8069589138031006, + "learning_rate": 0.0001317015167408831, + "loss": 0.22590700149536133, + "mean_token_accuracy": 0.9249501618742942, + "num_tokens": 3973168.0, + "step": 1700 + }, + { + "entropy": 0.2980187537521124, + "epoch": 4.353673723536737, + "grad_norm": 0.6587982773780823, + "learning_rate": 0.00012805564555811122, + "loss": 0.22620832443237304, + "mean_token_accuracy": 0.9248810124397278, + "num_tokens": 4092538.0, + "step": 1750 + }, + { + "entropy": 0.3055800000578165, + "epoch": 4.478206724782067, + "grad_norm": 0.8378816843032837, + "learning_rate": 0.00012434199832658783, + "loss": 0.23171533584594728, + "mean_token_accuracy": 0.9231083118915557, + "num_tokens": 4206935.0, + "step": 1800 + }, + { + "entropy": 0.31550996117293834, + "epoch": 4.602739726027397, + "grad_norm": 0.749613881111145, + "learning_rate": 0.00012056757403119691, + "loss": 0.23747362136840822, + "mean_token_accuracy": 0.9206199425458909, + "num_tokens": 4319440.0, + "step": 1850 + }, + { + "entropy": 0.30942291468381883, + "epoch": 4.7272727272727275, + "grad_norm": 0.7933925986289978, + "learning_rate": 0.0001167394862012918, + "loss": 0.23256269454956055, + "mean_token_accuracy": 0.9226748991012573, + "num_tokens": 4439513.0, + "step": 1900 + }, + { + "entropy": 0.3098080876469612, + "epoch": 4.851805728518057, + "grad_norm": 0.8441582918167114, + "learning_rate": 0.00011286494950406773, + "loss": 0.23220256805419923, + "mean_token_accuracy": 0.9222584775090218, + "num_tokens": 4562599.0, + "step": 1950 + }, + { + "entropy": 0.3098256988823414, + "epoch": 4.976338729763388, + "grad_norm": 0.8682230114936829, + "learning_rate": 0.00010895126614732359, + "loss": 0.23128936767578126, + "mean_token_accuracy": 0.923048702776432, + "num_tokens": 4677603.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.39352814238085304, + "eval_loss": 0.5585312247276306, + "eval_mean_token_accuracy": 0.8575408472571262, + "eval_num_tokens": 4700145.0, + "eval_runtime": 79.6426, + "eval_samples_per_second": 17.265, + "eval_steps_per_second": 2.16, + "step": 2010 + }, + { + "entropy": 0.242041623765471, + "epoch": 5.099626400996264, + "grad_norm": 0.8845140337944031, + "learning_rate": 0.00010500581211723872, + "loss": 0.16415899276733398, + "mean_token_accuracy": 0.9458679586949975, + "num_tokens": 4794835.0, + "step": 2050 + }, + { + "entropy": 0.22115826569497585, + "epoch": 5.224159402241594, + "grad_norm": 0.8571121692657471, + "learning_rate": 0.00010103602327710202, + "loss": 0.14506815910339355, + "mean_token_accuracy": 0.9509596079587936, + "num_tokens": 4912403.0, + "step": 2100 + }, + { + "entropy": 0.2277975185215473, + "epoch": 5.348692403486924, + "grad_norm": 0.6461961269378662, + "learning_rate": 9.7049381353193e-05, + "loss": 0.15208842277526854, + "mean_token_accuracy": 0.9484275713562965, + "num_tokens": 5025000.0, + "step": 2150 + }, + { + "entropy": 0.22508278906345366, + "epoch": 5.473225404732254, + "grad_norm": 0.6664603352546692, + "learning_rate": 9.305339983422608e-05, + "loss": 0.1507566452026367, + "mean_token_accuracy": 0.9494482669234275, + "num_tokens": 5142673.0, + "step": 2200 + }, + { + "entropy": 0.22525668181478978, + "epoch": 5.597758405977584, + "grad_norm": 0.6928425431251526, + "learning_rate": 8.905560981093345e-05, + "loss": 0.15167025566101075, + "mean_token_accuracy": 0.947735041975975, + "num_tokens": 5259364.0, + "step": 2250 + }, + { + "entropy": 0.21739320732653142, + "epoch": 5.722291407222914, + "grad_norm": 0.7611718773841858, + "learning_rate": 8.506354578247393e-05, + "loss": 0.1465705680847168, + "mean_token_accuracy": 0.9501141354441642, + "num_tokens": 5376956.0, + "step": 2300 + }, + { + "entropy": 0.2239777410775423, + "epoch": 5.846824408468244, + "grad_norm": 0.7929556965827942, + "learning_rate": 8.108473145641841e-05, + "loss": 0.14866115570068358, + "mean_token_accuracy": 0.9497014331817627, + "num_tokens": 5492490.0, + "step": 2350 + }, + { + "entropy": 0.21606657084077596, + "epoch": 5.971357409713574, + "grad_norm": 0.7851046323776245, + "learning_rate": 7.712666556907332e-05, + "loss": 0.14777716636657715, + "mean_token_accuracy": 0.9501199626922607, + "num_tokens": 5613602.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.32871103122137313, + "eval_loss": 0.635863184928894, + "eval_mean_token_accuracy": 0.8554029128579206, + "eval_num_tokens": 5640174.0, + "eval_runtime": 79.4073, + "eval_samples_per_second": 17.316, + "eval_steps_per_second": 2.166, + "step": 2412 + }, + { + "entropy": 0.18288949677588964, + "epoch": 6.094645080946451, + "grad_norm": 0.39350658655166626, + "learning_rate": 7.319680775286681e-05, + "loss": 0.11040553092956543, + "mean_token_accuracy": 0.9622839373169523, + "num_tokens": 5729864.0, + "step": 2450 + }, + { + "entropy": 0.16421406015753745, + "epoch": 6.219178082191781, + "grad_norm": 0.5661713480949402, + "learning_rate": 6.930256447743308e-05, + "loss": 0.0962723159790039, + "mean_token_accuracy": 0.967263371348381, + "num_tokens": 5845776.0, + "step": 2500 + }, + { + "entropy": 0.16072349682450293, + "epoch": 6.34371108343711, + "grad_norm": 0.755388617515564, + "learning_rate": 6.545127509088995e-05, + "loss": 0.09449175834655761, + "mean_token_accuracy": 0.9674956914782524, + "num_tokens": 5961333.0, + "step": 2550 + }, + { + "entropy": 0.15834762785583734, + "epoch": 6.468244084682441, + "grad_norm": 0.5315730571746826, + "learning_rate": 6.1650197987619e-05, + "loss": 0.09194418907165527, + "mean_token_accuracy": 0.9688713988661766, + "num_tokens": 6082253.0, + "step": 2600 + }, + { + "entropy": 0.16119305711239576, + "epoch": 6.592777085927771, + "grad_norm": 0.6552009582519531, + "learning_rate": 5.7906496928615814e-05, + "loss": 0.0957893180847168, + "mean_token_accuracy": 0.9680398201942444, + "num_tokens": 6200721.0, + "step": 2650 + }, + { + "entropy": 0.16668029114603997, + "epoch": 6.717310087173101, + "grad_norm": 0.6891521215438843, + "learning_rate": 5.4227227540192906e-05, + "loss": 0.09778289794921875, + "mean_token_accuracy": 0.9666138800978661, + "num_tokens": 6318066.0, + "step": 2700 + }, + { + "entropy": 0.1648960181325674, + "epoch": 6.841843088418431, + "grad_norm": 0.5747121572494507, + "learning_rate": 5.061932401648035e-05, + "loss": 0.09770219802856445, + "mean_token_accuracy": 0.9660053381323814, + "num_tokens": 6430978.0, + "step": 2750 + }, + { + "entropy": 0.15946010805666447, + "epoch": 6.966376089663761, + "grad_norm": 0.6286085247993469, + "learning_rate": 4.7089586050785194e-05, + "loss": 0.09402350425720214, + "mean_token_accuracy": 0.9677979084849357, + "num_tokens": 6549781.0, + "step": 2800 + }, + { + "epoch": 7.0, + "eval_entropy": 0.2758399397134781, + "eval_loss": 0.7382693290710449, + "eval_mean_token_accuracy": 0.8535292623348014, + "eval_num_tokens": 6580203.0, + "eval_runtime": 79.5655, + "eval_samples_per_second": 17.281, + "eval_steps_per_second": 2.162, + "step": 2814 + }, + { + "entropy": 0.15347467818193966, + "epoch": 7.089663760896638, + "grad_norm": 0.45861202478408813, + "learning_rate": 4.36446660204407e-05, + "loss": 0.08010212898254394, + "mean_token_accuracy": 0.9720996783839332, + "num_tokens": 6661238.0, + "step": 2850 + }, + { + "entropy": 0.1345258642360568, + "epoch": 7.214196762141968, + "grad_norm": 0.4790540933609009, + "learning_rate": 4.029105644929618e-05, + "loss": 0.07323225498199464, + "mean_token_accuracy": 0.9749647602438927, + "num_tokens": 6774391.0, + "step": 2900 + }, + { + "entropy": 0.13495642982423306, + "epoch": 7.338729763387297, + "grad_norm": 0.41177162528038025, + "learning_rate": 3.7035077771478046e-05, + "loss": 0.0723848009109497, + "mean_token_accuracy": 0.9750138682126999, + "num_tokens": 6892967.0, + "step": 2950 + }, + { + "entropy": 0.14119064182043076, + "epoch": 7.463262764632628, + "grad_norm": 0.3276011049747467, + "learning_rate": 3.3882866419481806e-05, + "loss": 0.07503739356994629, + "mean_token_accuracy": 0.973833509683609, + "num_tokens": 7006692.0, + "step": 3000 + }, + { + "entropy": 0.13939726265147329, + "epoch": 7.587795765877957, + "grad_norm": 0.29367685317993164, + "learning_rate": 3.084036325904632e-05, + "loss": 0.07321533203125, + "mean_token_accuracy": 0.9747413221001625, + "num_tokens": 7124768.0, + "step": 3050 + }, + { + "entropy": 0.1330118723027408, + "epoch": 7.712328767123288, + "grad_norm": 0.26096275448799133, + "learning_rate": 2.7913302392605573e-05, + "loss": 0.0725630760192871, + "mean_token_accuracy": 0.9755767813324928, + "num_tokens": 7242764.0, + "step": 3100 + }, + { + "entropy": 0.12999726412817836, + "epoch": 7.8368617683686175, + "grad_norm": 0.4909002482891083, + "learning_rate": 2.5107200352420576e-05, + "loss": 0.06970284938812256, + "mean_token_accuracy": 0.9758264967799186, + "num_tokens": 7366604.0, + "step": 3150 + }, + { + "entropy": 0.13837616711854936, + "epoch": 7.961394769613948, + "grad_norm": 0.37932804226875305, + "learning_rate": 2.2427345703758274e-05, + "loss": 0.07299811363220216, + "mean_token_accuracy": 0.9738594210147857, + "num_tokens": 7481687.0, + "step": 3200 + }, + { + "epoch": 8.0, + "eval_entropy": 0.25052128273040747, + "eval_loss": 0.8225418925285339, + "eval_mean_token_accuracy": 0.8537235357040582, + "eval_num_tokens": 7520232.0, + "eval_runtime": 79.595, + "eval_samples_per_second": 17.275, + "eval_steps_per_second": 2.161, + "step": 3216 + }, + { + "entropy": 0.12771825577047738, + "epoch": 8.084682440846825, + "grad_norm": 0.1904180347919464, + "learning_rate": 1.9878789077711986e-05, + "loss": 0.06388124942779541, + "mean_token_accuracy": 0.9773780998557505, + "num_tokens": 7601281.0, + "step": 3250 + }, + { + "entropy": 0.12670317662879824, + "epoch": 8.209215442092155, + "grad_norm": 0.31729868054389954, + "learning_rate": 1.7466333652448554e-05, + "loss": 0.06243476390838623, + "mean_token_accuracy": 0.977044507265091, + "num_tokens": 7720929.0, + "step": 3300 + }, + { + "entropy": 0.12671989548951387, + "epoch": 8.333748443337484, + "grad_norm": 0.2833121120929718, + "learning_rate": 1.5194526100821374e-05, + "loss": 0.06502001285552979, + "mean_token_accuracy": 0.9766163811087608, + "num_tokens": 7834637.0, + "step": 3350 + }, + { + "entropy": 0.12553330410271882, + "epoch": 8.458281444582815, + "grad_norm": 0.3116062581539154, + "learning_rate": 1.3067648021410455e-05, + "loss": 0.06511738300323486, + "mean_token_accuracy": 0.9765481826663017, + "num_tokens": 7947969.0, + "step": 3400 + }, + { + "entropy": 0.12742114875465632, + "epoch": 8.582814445828145, + "grad_norm": 0.18022380769252777, + "learning_rate": 1.1089707869138902e-05, + "loss": 0.06584225654602051, + "mean_token_accuracy": 0.9765808299183846, + "num_tokens": 8063392.0, + "step": 3450 + }, + { + "entropy": 0.1201029148697853, + "epoch": 8.707347447073474, + "grad_norm": 0.20968234539031982, + "learning_rate": 9.26443340067398e-06, + "loss": 0.06199615478515625, + "mean_token_accuracy": 0.9771515518426895, + "num_tokens": 8186140.0, + "step": 3500 + }, + { + "entropy": 0.1267964872159064, + "epoch": 8.831880448318804, + "grad_norm": 0.29122787714004517, + "learning_rate": 7.5952646488506875e-06, + "loss": 0.0641715908050537, + "mean_token_accuracy": 0.9764416518807412, + "num_tokens": 8304658.0, + "step": 3550 + }, + { + "entropy": 0.13317630462348462, + "epoch": 8.956413449564135, + "grad_norm": 0.2513009309768677, + "learning_rate": 6.0853474393585395e-06, + "loss": 0.06596921443939209, + "mean_token_accuracy": 0.9751221585273743, + "num_tokens": 8419672.0, + "step": 3600 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2314538847395154, + "eval_loss": 0.9136112332344055, + "eval_mean_token_accuracy": 0.8535278571206469, + "eval_num_tokens": 8460261.0, + "eval_runtime": 79.6191, + "eval_samples_per_second": 17.27, + "eval_steps_per_second": 2.16, + "step": 3618 + }, + { + "entropy": 0.12940765023607798, + "epoch": 9.079701120797012, + "grad_norm": 0.2125224620103836, + "learning_rate": 4.7375274619107725e-06, + "loss": 0.0633786964416504, + "mean_token_accuracy": 0.976845854460591, + "num_tokens": 8531214.0, + "step": 3650 + }, + { + "entropy": 0.11837937209755182, + "epoch": 9.204234122042342, + "grad_norm": 0.2279079407453537, + "learning_rate": 3.5543449070695066e-06, + "loss": 0.059304299354553225, + "mean_token_accuracy": 0.9785094287991524, + "num_tokens": 8652483.0, + "step": 3700 + }, + { + "entropy": 0.12689687427133323, + "epoch": 9.32876712328767, + "grad_norm": 0.16726821660995483, + "learning_rate": 2.5380296788347564e-06, + "loss": 0.062330703735351566, + "mean_token_accuracy": 0.9772107803821564, + "num_tokens": 8767142.0, + "step": 3750 + }, + { + "entropy": 0.11406100125983358, + "epoch": 9.453300124533001, + "grad_norm": 0.2180442363023758, + "learning_rate": 1.6904971920201498e-06, + "loss": 0.0574225664138794, + "mean_token_accuracy": 0.9796162959933281, + "num_tokens": 8889508.0, + "step": 3800 + }, + { + "entropy": 0.12276196179911494, + "epoch": 9.577833125778332, + "grad_norm": 0.2879609167575836, + "learning_rate": 1.013344762335574e-06, + "loss": 0.06335700511932373, + "mean_token_accuracy": 0.9771510726213455, + "num_tokens": 9000626.0, + "step": 3850 + }, + { + "entropy": 0.1240639734454453, + "epoch": 9.70236612702366, + "grad_norm": 0.24199265241622925, + "learning_rate": 5.078485959805103e-07, + "loss": 0.060325398445129394, + "mean_token_accuracy": 0.9777579417824745, + "num_tokens": 9118844.0, + "step": 3900 + }, + { + "entropy": 0.11595728505402804, + "epoch": 9.826899128268991, + "grad_norm": 0.17497791349887848, + "learning_rate": 1.7496138442149407e-07, + "loss": 0.058862648010253905, + "mean_token_accuracy": 0.9790140253305435, + "num_tokens": 9240716.0, + "step": 3950 + }, + { + "entropy": 0.12840898344293236, + "epoch": 9.951432129514322, + "grad_norm": 0.2691340744495392, + "learning_rate": 1.5310508886877188e-08, + "loss": 0.06197177886962891, + "mean_token_accuracy": 0.9764682227373123, + "num_tokens": 9354788.0, + "step": 4000 + }, + { + "epoch": 10.0, + "eval_entropy": 0.22721278572151826, + "eval_loss": 0.9521453976631165, + "eval_mean_token_accuracy": 0.853252439997917, + "eval_num_tokens": 9400290.0, + "eval_runtime": 79.7213, + "eval_samples_per_second": 17.248, + "eval_steps_per_second": 2.158, + "step": 4020 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3.8801038188323635e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/README.md b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/adapter_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd48592ebfde6ffa9e1fa29412808c550e6c8898 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05833860659003597, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "k_proj", + "q_proj", + "o_proj", + "v_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/chat_template.jinja b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/tokenizer_config.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/trainer_state.json b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b086c995296bc9a7093e33034e35e966e90a37fb --- /dev/null +++ b/random_original_Swedish/Qwen3.5-4B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-804/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 804, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5681732833385467, + "epoch": 0.12453300124533001, + "grad_norm": 1.2423518896102905, + "learning_rate": 2.245073006787377e-05, + "loss": 1.3567063903808594, + "mean_token_accuracy": 0.7048975314199925, + "num_tokens": 114583.0, + "step": 50 + }, + { + "entropy": 0.7356800127029419, + "epoch": 0.24906600249066002, + "grad_norm": 1.0828187465667725, + "learning_rate": 4.535963830039802e-05, + "loss": 0.65197021484375, + "mean_token_accuracy": 0.8213974100351333, + "num_tokens": 234541.0, + "step": 100 + }, + { + "entropy": 0.6838073092699051, + "epoch": 0.37359900373599003, + "grad_norm": 0.9256018996238708, + "learning_rate": 6.826854653292228e-05, + "loss": 0.6043741607666016, + "mean_token_accuracy": 0.8296174100041389, + "num_tokens": 351327.0, + "step": 150 + }, + { + "entropy": 0.6625622838735581, + "epoch": 0.49813200498132004, + "grad_norm": 0.8982412815093994, + "learning_rate": 9.117745476544654e-05, + "loss": 0.5804051971435547, + "mean_token_accuracy": 0.8364648693799972, + "num_tokens": 472595.0, + "step": 200 + }, + { + "entropy": 0.6394155050814152, + "epoch": 0.6226650062266501, + "grad_norm": 0.6671016812324524, + "learning_rate": 0.0001140863629979708, + "loss": 0.5582684326171875, + "mean_token_accuracy": 0.8420979034900665, + "num_tokens": 590160.0, + "step": 250 + }, + { + "entropy": 0.6368238939344883, + "epoch": 0.7471980074719801, + "grad_norm": 0.6754769086837769, + "learning_rate": 0.00013699527123049504, + "loss": 0.5563363265991211, + "mean_token_accuracy": 0.8401645755767823, + "num_tokens": 706659.0, + "step": 300 + }, + { + "entropy": 0.6217501659691334, + "epoch": 0.8717310087173101, + "grad_norm": 0.714057445526123, + "learning_rate": 0.0001599041794630193, + "loss": 0.5475285339355469, + "mean_token_accuracy": 0.8440281334519386, + "num_tokens": 821763.0, + "step": 350 + }, + { + "entropy": 0.6209351749718189, + "epoch": 0.9962640099626401, + "grad_norm": 0.7717300057411194, + "learning_rate": 0.00018281308769554355, + "loss": 0.5463589477539063, + "mean_token_accuracy": 0.8433159792423248, + "num_tokens": 936946.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.565151353561601, + "eval_loss": 0.5447193384170532, + "eval_mean_token_accuracy": 0.8464361508918363, + "eval_num_tokens": 940029.0, + "eval_runtime": 80.9151, + "eval_samples_per_second": 16.993, + "eval_steps_per_second": 2.126, + "step": 402 + }, + { + "entropy": 0.5661561889479859, + "epoch": 1.1195516811955168, + "grad_norm": 0.708380937576294, + "learning_rate": 0.00018411093929332744, + "loss": 0.4969264221191406, + "mean_token_accuracy": 0.8550105146085373, + "num_tokens": 1054389.0, + "step": 450 + }, + { + "entropy": 0.5690013617277145, + "epoch": 1.244084682440847, + "grad_norm": 0.8610317707061768, + "learning_rate": 0.00018386114730685091, + "loss": 0.49536712646484377, + "mean_token_accuracy": 0.8549688473343849, + "num_tokens": 1170301.0, + "step": 500 + }, + { + "entropy": 0.5661576768755913, + "epoch": 1.3686176836861768, + "grad_norm": 0.8410939574241638, + "learning_rate": 0.00018343840453669726, + "loss": 0.4957312774658203, + "mean_token_accuracy": 0.8546139162778854, + "num_tokens": 1284497.0, + "step": 550 + }, + { + "entropy": 0.5658727620542049, + "epoch": 1.4931506849315068, + "grad_norm": 0.6218363046646118, + "learning_rate": 0.00018284350771181608, + "loss": 0.4974911499023438, + "mean_token_accuracy": 0.855501549243927, + "num_tokens": 1401566.0, + "step": 600 + }, + { + "entropy": 0.5707372877001763, + "epoch": 1.6176836861768369, + "grad_norm": 0.8585225343704224, + "learning_rate": 0.0001820775780140605, + "loss": 0.49875095367431643, + "mean_token_accuracy": 0.8563189420104027, + "num_tokens": 1515959.0, + "step": 650 + }, + { + "entropy": 0.5534089787304401, + "epoch": 1.7422166874221667, + "grad_norm": 0.6285793781280518, + "learning_rate": 0.00018114205896513374, + "loss": 0.48918685913085935, + "mean_token_accuracy": 0.8578527864813804, + "num_tokens": 1638116.0, + "step": 700 + }, + { + "entropy": 0.5458812513947486, + "epoch": 1.8667496886674968, + "grad_norm": 0.71883225440979, + "learning_rate": 0.00018003871370603276, + "loss": 0.482542839050293, + "mean_token_accuracy": 0.8564121940732002, + "num_tokens": 1754423.0, + "step": 750 + }, + { + "entropy": 0.5502609372138977, + "epoch": 1.9912826899128269, + "grad_norm": 0.6442635655403137, + "learning_rate": 0.00017876962167411625, + "loss": 0.48193965911865233, + "mean_token_accuracy": 0.8567470651865006, + "num_tokens": 1871167.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5426353398104047, + "eval_loss": 0.5083975791931152, + "eval_mean_token_accuracy": 0.8554168769786524, + "eval_num_tokens": 1880058.0, + "eval_runtime": 79.8868, + "eval_samples_per_second": 17.212, + "eval_steps_per_second": 2.153, + "step": 804 + } + ], + "logging_steps": 50, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.764884851111526e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}