Add files using upload-large-folder tool

exp2b/swap_0.7_last_to_hit_5039/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_hit_5039/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 29.146338619402986,
+    "total_flos": 2.089917666164736e+18,
+    "train_loss": 0.634907502746582,
+    "train_runtime": 39649.7458,
+    "train_samples": 274422,
+    "train_samples_per_second": 346.058,
+    "train_steps_per_second": 4.327
+}

exp2b/swap_0.7_last_to_push_1032/README.md

@@ -0,0 +1,163 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.7_last_to_push_1032
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/dehrdvzz)
+# exceptions_exp2_swap_0.7_last_to_push_1032
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5644
+- Accuracy: 0.3686
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 1032
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Accuracy | Validation Loss |
+|:-------------:|:-------:|:------:|:--------:|:---------------:|
+| 4.8199        | 0.2915  | 1000   | 0.2557   | 4.7489          |
+| 4.3394        | 0.5830  | 2000   | 0.2985   | 4.2872          |
+| 4.152         | 0.8745  | 3000   | 0.3142   | 4.1027          |
+| 3.998         | 1.1659  | 4000   | 0.3234   | 3.9996          |
+| 3.9535        | 1.4574  | 5000   | 0.3306   | 3.9238          |
+| 3.8885        | 1.7489  | 6000   | 0.3357   | 3.8655          |
+| 3.7583        | 2.0402  | 7000   | 0.3400   | 3.8238          |
+| 3.7615        | 2.3317  | 8000   | 0.3430   | 3.7939          |
+| 3.7454        | 2.6233  | 9000   | 0.3456   | 3.7623          |
+| 3.7305        | 2.9148  | 10000  | 0.3484   | 3.7364          |
+| 3.6492        | 3.2061  | 11000  | 0.3502   | 3.7243          |
+| 3.65          | 3.4976  | 12000  | 0.3517   | 3.7054          |
+| 3.6525        | 3.7891  | 13000  | 0.3534   | 3.6861          |
+| 3.5452        | 4.0805  | 14000  | 0.3549   | 3.6810          |
+| 3.5745        | 4.3720  | 15000  | 0.3562   | 3.6666          |
+| 3.5906        | 4.6635  | 16000  | 0.3571   | 3.6557          |
+| 3.5889        | 4.9550  | 17000  | 0.3584   | 3.6405          |
+| 3.5194        | 5.2463  | 18000  | 0.3590   | 3.6444          |
+| 3.5288        | 5.5378  | 19000  | 0.3600   | 3.6323          |
+| 3.5341        | 5.8293  | 20000  | 0.3609   | 3.6224          |
+| 3.4369        | 6.1207  | 21000  | 0.3612   | 3.6253          |
+| 3.4869        | 6.4122  | 22000  | 0.3621   | 3.6204          |
+| 3.4957        | 6.7037  | 23000  | 0.3627   | 3.6091          |
+| 3.5051        | 6.9952  | 24000  | 0.3635   | 3.6000          |
+| 3.4353        | 7.2866  | 25000  | 0.3633   | 3.6091          |
+| 3.4666        | 7.5781  | 26000  | 0.3642   | 3.5999          |
+| 3.4623        | 7.8696  | 27000  | 0.3650   | 3.5864          |
+| 3.3899        | 8.1609  | 28000  | 0.3647   | 3.5995          |
+| 3.4279        | 8.4524  | 29000  | 0.3652   | 3.5901          |
+| 3.434         | 8.7439  | 30000  | 0.3659   | 3.5819          |
+| 3.3392        | 9.0353  | 31000  | 0.3660   | 3.5891          |
+| 3.3891        | 9.3268  | 32000  | 0.3659   | 3.5865          |
+| 3.3997        | 9.6183  | 33000  | 0.3668   | 3.5792          |
+| 3.4287        | 9.9098  | 34000  | 0.3676   | 3.5705          |
+| 3.3463        | 10.2011 | 35000  | 0.3673   | 3.5800          |
+| 3.3681        | 10.4927 | 36000  | 0.3674   | 3.5784          |
+| 3.4045        | 10.7842 | 37000  | 0.3680   | 3.5678          |
+| 3.2969        | 11.0755 | 38000  | 0.3678   | 3.5783          |
+| 3.3605        | 11.3670 | 39000  | 0.3681   | 3.5736          |
+| 3.3652        | 11.6585 | 40000  | 0.3686   | 3.5644          |
+| 3.3896        | 11.9500 | 41000  | 0.3690   | 3.5596          |
+| 3.3144        | 12.2414 | 42000  | 0.3682   | 3.5734          |
+| 3.3353        | 12.5329 | 43000  | 0.3688   | 3.5665          |
+| 3.3535        | 12.8244 | 44000  | 0.3695   | 3.5587          |
+| 3.272         | 13.1157 | 45000  | 0.3693   | 3.5701          |
+| 3.3143        | 13.4072 | 46000  | 0.3690   | 3.5667          |
+| 3.3453        | 13.6988 | 47000  | 0.3699   | 3.5564          |
+| 3.355         | 13.9903 | 48000  | 0.3706   | 3.5493          |
+| 3.2878        | 14.2816 | 49000  | 0.3700   | 3.5643          |
+| 3.3134        | 14.5731 | 50000  | 0.3702   | 3.5556          |
+| 3.3318        | 14.8646 | 51000  | 0.3706   | 3.5504          |
+| 3.2652        | 15.1560 | 52000  | 0.3700   | 3.5669          |
+| 3.2886        | 15.4475 | 53000  | 0.3706   | 3.5605          |
+| 3.3238        | 15.7390 | 54000  | 0.3711   | 3.5493          |
+| 3.2186        | 16.0303 | 55000  | 0.3705   | 3.5647          |
+| 3.2666        | 16.3218 | 56000  | 0.3708   | 3.5616          |
+| 3.2879        | 16.6133 | 57000  | 0.3713   | 3.5543          |
+| 3.3007        | 16.9049 | 58000  | 0.3717   | 3.5442          |
+| 3.234         | 17.1962 | 59000  | 0.3706   | 3.5637          |
+| 3.2701        | 17.4877 | 60000  | 0.3714   | 3.5541          |
+| 3.2802        | 17.7792 | 61000  | 0.3720   | 3.5504          |
+| 3.1942        | 18.0705 | 62000  | 0.3707   | 3.5671          |
+| 3.2393        | 18.3621 | 63000  | 0.3711   | 3.5591          |
+| 3.2598        | 18.6536 | 64000  | 0.3718   | 3.5515          |
+| 3.271         | 18.9451 | 65000  | 0.3721   | 3.5439          |
+| 3.2108        | 19.2364 | 66000  | 0.3715   | 3.5616          |
+| 3.256         | 19.5279 | 67000  | 0.3719   | 3.5521          |
+| 3.2692        | 19.8194 | 68000  | 0.3723   | 3.5431          |
+| 3.1916        | 20.1108 | 69000  | 0.3715   | 3.5643          |
+| 3.2157        | 20.4023 | 70000  | 0.3720   | 3.5574          |
+| 3.2554        | 20.6938 | 71000  | 0.3724   | 3.5460          |
+| 3.2615        | 20.9853 | 72000  | 0.3726   | 3.5405          |
+| 3.2016        | 21.2766 | 73000  | 0.3719   | 3.5635          |
+| 3.2166        | 21.5682 | 74000  | 0.3722   | 3.5502          |
+| 3.243         | 21.8597 | 75000  | 0.3728   | 3.5441          |
+| 3.1716        | 22.1510 | 76000  | 0.3721   | 3.5593          |
+| 3.1998        | 22.4425 | 77000  | 0.3724   | 3.5544          |
+| 3.2394        | 22.7340 | 78000  | 0.3729   | 3.5470          |
+| 3.1309        | 23.0254 | 79000  | 0.3722   | 3.5588          |
+| 3.1858        | 23.3169 | 80000  | 0.3722   | 3.5583          |
+| 3.1992        | 23.6084 | 81000  | 3.5650   | 0.3722          |
+| 3.2197        | 23.8999 | 82000  | 3.5497   | 0.3726          |
+| 3.1587        | 24.1915 | 83000  | 3.5615   | 0.3723          |
+| 3.1985        | 24.4830 | 84000  | 3.5550   | 0.3726          |
+| 3.2164        | 24.7745 | 85000  | 3.5455   | 0.3733          |
+| 3.1432        | 25.0659 | 86000  | 3.5623   | 0.3723          |
+| 3.1808        | 25.3574 | 87000  | 3.5566   | 0.3728          |
+| 3.2007        | 25.6489 | 88000  | 3.5481   | 0.3731          |
+| 3.205         | 25.9404 | 89000  | 3.5415   | 0.3737          |
+| 3.1654        | 26.2318 | 90000  | 3.5589   | 0.3727          |
+| 3.1871        | 26.5233 | 91000  | 3.5475   | 0.3732          |
+| 3.1968        | 26.8148 | 92000  | 3.5477   | 0.3735          |
+| 3.1321        | 27.1061 | 93000  | 3.5612   | 0.3728          |
+| 3.1537        | 27.3976 | 94000  | 3.5578   | 0.3731          |
+| 3.1954        | 27.6891 | 95000  | 3.5478   | 0.3735          |
+| 3.2019        | 27.9806 | 96000  | 3.5409   | 0.3739          |
+| 3.1278        | 28.2720 | 97000  | 3.5578   | 0.3732          |
+| 3.1623        | 28.5635 | 98000  | 3.5488   | 0.3737          |
+| 3.1833        | 28.8550 | 99000  | 3.5416   | 0.3744          |
+| 3.1149        | 29.1463 | 100000 | 3.5644   | 0.3731          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.7_last_to_push_1032/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 29.146338619402986,
+    "eval_accuracy": 0.3686261364670224,
+    "eval_loss": 3.5644266605377197,
+    "eval_runtime": 96.6994,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 171.945,
+    "eval_steps_per_second": 10.755,
+    "perplexity": 35.31919770831738,
+    "total_flos": 2.089917666164736e+18,
+    "train_loss": 0.6348209342956543,
+    "train_runtime": 14939.2063,
+    "train_samples": 274422,
+    "train_samples_per_second": 918.462,
+    "train_steps_per_second": 11.483
+}

exp2b/swap_0.7_last_to_push_1032/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.7_last_to_push_1032/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 29.146338619402986,
+    "eval_accuracy": 0.3686261364670224,
+    "eval_loss": 3.5644266605377197,
+    "eval_runtime": 96.6994,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 171.945,
+    "eval_steps_per_second": 10.755,
+    "perplexity": 35.31919770831738
+}

exp2b/swap_0.7_last_to_push_1032/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.7_last_to_push_1032/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_push_1032/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_push_1032/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 29.146338619402986,
+    "total_flos": 2.089917666164736e+18,
+    "train_loss": 0.6348209342956543,
+    "train_runtime": 14939.2063,
+    "train_samples": 274422,
+    "train_samples_per_second": 918.462,
+    "train_steps_per_second": 11.483
+}

exp2b/swap_0.7_last_to_push_2128/README.md

@@ -0,0 +1,179 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.7_last_to_push_2128
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/3emucsk7)
+# exceptions_exp2_swap_0.7_last_to_push_2128
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5665
+- Accuracy: 0.3685
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 2128
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Accuracy | Validation Loss |
+|:-------------:|:-------:|:------:|:--------:|:---------------:|
+| 4.8405        | 0.2915  | 1000   | 0.2538   | 4.7545          |
+| 4.3421        | 0.5830  | 2000   | 0.2983   | 4.2930          |
+| 4.1511        | 0.8745  | 3000   | 0.3143   | 4.1041          |
+| 4.0075        | 1.1659  | 4000   | 0.3236   | 4.0015          |
+| 3.9398        | 1.4574  | 5000   | 0.3304   | 3.9234          |
+| 3.8836        | 1.7489  | 6000   | 0.3355   | 3.8672          |
+| 3.7667        | 2.0402  | 7000   | 0.3395   | 3.8254          |
+| 3.7647        | 2.3317  | 8000   | 0.3430   | 3.7946          |
+| 3.7476        | 2.6233  | 9000   | 0.3456   | 3.7627          |
+| 3.7253        | 2.9148  | 10000  | 0.3480   | 3.7392          |
+| 3.6522        | 3.2061  | 11000  | 0.3497   | 3.7253          |
+| 3.66          | 3.4976  | 12000  | 0.3516   | 3.7062          |
+| 3.6552        | 3.7891  | 13000  | 0.3531   | 3.6898          |
+| 3.5443        | 4.0805  | 14000  | 0.3548   | 3.6803          |
+| 3.5694        | 4.3720  | 15000  | 0.3561   | 3.6704          |
+| 3.5998        | 4.6635  | 16000  | 0.3568   | 3.6596          |
+| 3.5871        | 4.9550  | 17000  | 0.3585   | 3.6424          |
+| 3.4993        | 5.2463  | 18000  | 0.3590   | 3.6442          |
+| 3.5276        | 5.5378  | 19000  | 0.3596   | 3.6372          |
+| 3.5397        | 5.8293  | 20000  | 0.3607   | 3.6235          |
+| 3.4575        | 6.1207  | 21000  | 0.3610   | 3.6287          |
+| 3.4711        | 6.4122  | 22000  | 0.3618   | 3.6190          |
+| 3.4998        | 6.7037  | 23000  | 0.3626   | 3.6091          |
+| 3.5047        | 6.9952  | 24000  | 0.3634   | 3.6004          |
+| 3.4571        | 7.2866  | 25000  | 0.3633   | 3.6089          |
+| 3.4564        | 7.5781  | 26000  | 0.3641   | 3.6010          |
+| 3.4619        | 7.8696  | 27000  | 0.3645   | 3.5920          |
+| 3.3915        | 8.1609  | 28000  | 0.3646   | 3.5990          |
+| 3.4238        | 8.4524  | 29000  | 0.3648   | 3.5954          |
+| 3.433         | 8.7439  | 30000  | 0.3657   | 3.5863          |
+| 3.3406        | 9.0353  | 31000  | 0.3655   | 3.5907          |
+| 3.3767        | 9.3268  | 32000  | 0.3661   | 3.5872          |
+| 3.4124        | 9.6183  | 33000  | 0.3664   | 3.5781          |
+| 3.4173        | 9.9098  | 34000  | 0.3671   | 3.5719          |
+| 3.353         | 10.2011 | 35000  | 0.3668   | 3.5851          |
+| 3.386         | 10.4927 | 36000  | 0.3673   | 3.5757          |
+| 3.3967        | 10.7842 | 37000  | 0.3681   | 3.5682          |
+| 3.3008        | 11.0755 | 38000  | 0.3674   | 3.5771          |
+| 3.3421        | 11.3670 | 39000  | 0.3676   | 3.5748          |
+| 3.3865        | 11.6585 | 40000  | 0.3685   | 3.5665          |
+| 3.3826        | 11.9500 | 41000  | 0.3691   | 3.5591          |
+| 3.3125        | 12.2414 | 42000  | 0.3687   | 3.5711          |
+| 3.3482        | 12.5329 | 43000  | 0.3687   | 3.5677          |
+| 3.3548        | 12.8244 | 44000  | 0.3694   | 3.5561          |
+| 3.2723        | 13.1157 | 45000  | 0.3688   | 3.5734          |
+| 3.3128        | 13.4072 | 46000  | 0.3696   | 3.5666          |
+| 3.3554        | 13.6988 | 47000  | 0.3700   | 3.5555          |
+| 3.3418        | 13.9903 | 48000  | 0.3703   | 3.5509          |
+| 3.2873        | 14.2816 | 49000  | 0.3699   | 3.5642          |
+| 3.3214        | 14.5731 | 50000  | 0.3699   | 3.5598          |
+| 3.3372        | 14.8646 | 51000  | 0.3704   | 3.5496          |
+| 3.2634        | 15.1560 | 52000  | 0.3700   | 3.5655          |
+| 3.3005        | 15.4475 | 53000  | 0.3703   | 3.5598          |
+| 3.3022        | 15.7390 | 54000  | 0.3707   | 3.5546          |
+| 3.2172        | 16.0303 | 55000  | 0.3706   | 3.5590          |
+| 3.2679        | 16.3218 | 56000  | 0.3707   | 3.5607          |
+| 3.2927        | 16.6133 | 57000  | 0.3713   | 3.5518          |
+| 3.3154        | 16.9049 | 58000  | 0.3717   | 3.5446          |
+| 3.2442        | 17.1962 | 59000  | 0.3711   | 3.5609          |
+| 3.2761        | 17.4877 | 60000  | 0.3714   | 3.5523          |
+| 3.291         | 17.7792 | 61000  | 0.3719   | 3.5461          |
+| 3.1997        | 18.0705 | 62000  | 0.3712   | 3.5632          |
+| 3.2501        | 18.3621 | 63000  | 0.3716   | 3.5541          |
+| 3.2619        | 18.6536 | 64000  | 0.3718   | 3.5468          |
+| 3.2919        | 18.9451 | 65000  | 0.3722   | 3.5423          |
+| 3.2264        | 19.2364 | 66000  | 0.3715   | 3.5561          |
+| 3.2394        | 19.5279 | 67000  | 0.3720   | 3.5495          |
+| 3.2665        | 19.8194 | 68000  | 0.3725   | 3.5456          |
+| 3.1939        | 20.1108 | 69000  | 0.3716   | 3.5588          |
+| 3.2149        | 20.4023 | 70000  | 0.3719   | 3.5556          |
+| 3.2447        | 20.6938 | 71000  | 0.3722   | 3.5452          |
+| 3.2733        | 20.9853 | 72000  | 0.3730   | 3.5387          |
+| 3.2131        | 21.2766 | 73000  | 0.3721   | 3.5581          |
+| 3.2312        | 21.5682 | 74000  | 0.3727   | 3.5495          |
+| 3.2569        | 21.8597 | 75000  | 0.3731   | 3.5411          |
+| 3.1789        | 22.1510 | 76000  | 0.3719   | 3.5589          |
+| 3.202         | 22.4425 | 77000  | 0.3724   | 3.5496          |
+| 3.2272        | 22.7340 | 78000  | 0.3730   | 3.5428          |
+| 3.1514        | 23.0254 | 79000  | 0.3726   | 3.5549          |
+| 3.1818        | 23.3169 | 80000  | 0.3724   | 3.5531          |
+| 3.1943        | 23.6084 | 81000  | 3.5582   | 0.3727          |
+| 3.1961        | 23.8999 | 82000  | 3.5507   | 0.3727          |
+| 3.1603        | 24.1915 | 83000  | 3.5631   | 0.3723          |
+| 3.1921        | 24.4830 | 84000  | 3.5538   | 0.3728          |
+| 3.2218        | 24.7745 | 85000  | 3.5425   | 0.3734          |
+| 3.1492        | 25.0659 | 86000  | 3.5576   | 0.3726          |
+| 3.1824        | 25.3574 | 87000  | 3.5505   | 0.3731          |
+| 3.201         | 25.6489 | 88000  | 3.5468   | 0.3735          |
+| 3.2114        | 25.9404 | 89000  | 3.5391   | 0.3738          |
+| 3.1724        | 26.2318 | 90000  | 3.5586   | 0.3729          |
+| 3.1831        | 26.5233 | 91000  | 3.5528   | 0.3733          |
+| 3.2083        | 26.8148 | 92000  | 3.5445   | 0.3736          |
+| 3.1286        | 27.1061 | 93000  | 3.5576   | 0.3732          |
+| 3.1577        | 27.3976 | 94000  | 3.5532   | 0.3733          |
+| 3.1856        | 27.6891 | 95000  | 3.5450   | 0.3739          |
+| 3.211         | 27.9806 | 96000  | 3.5370   | 0.3744          |
+| 3.1377        | 28.2720 | 97000  | 3.5558   | 0.3734          |
+| 3.1703        | 28.5635 | 98000  | 3.5500   | 0.3738          |
+| 3.1761        | 28.8550 | 99000  | 3.5446   | 0.3739          |
+| 3.1259        | 29.1463 | 100000 | 3.5599   | 0.3731          |
+| 3.1412        | 29.4378 | 101000 | 3.5544   | 0.3736          |
+| 3.1537        | 29.7294 | 102000 | 3.5457   | 0.3741          |
+| 3.0842        | 30.0207 | 103000 | 3.5555   | 0.3737          |
+| 3.1355        | 30.3122 | 104000 | 3.5562   | 0.3735          |
+| 3.1478        | 30.6037 | 105000 | 3.5494   | 0.3740          |
+| 3.166         | 30.8952 | 106000 | 3.5430   | 0.3743          |
+| 3.113         | 31.1866 | 107000 | 3.5579   | 0.3739          |
+| 3.1249        | 31.4781 | 108000 | 3.5542   | 0.3737          |
+| 3.1627        | 31.7696 | 109000 | 3.5448   | 0.3745          |
+| 3.0785        | 32.0609 | 110000 | 3.5565   | 0.3738          |
+| 3.1107        | 32.3524 | 111000 | 3.5553   | 0.3741          |
+| 3.1327        | 32.6439 | 112000 | 3.5484   | 0.3742          |
+| 3.1514        | 32.9355 | 113000 | 3.5464   | 0.3744          |
+| 3.1022        | 33.2268 | 114000 | 3.5567   | 0.3740          |
+| 3.1056        | 33.5183 | 115000 | 3.5551   | 0.3739          |
+| 3.1411        | 33.8098 | 116000 | 3.5420   | 0.3748          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.7_last_to_push_2128/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 33.809818097014926,
+    "eval_accuracy": 0.36852503478827553,
+    "eval_loss": 3.566509485244751,
+    "eval_runtime": 81.1363,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 204.927,
+    "eval_steps_per_second": 12.818,
+    "perplexity": 35.39283806929647,
+    "total_flos": 2.424310847373312e+18,
+    "train_loss": 0.9799438093777361,
+    "train_runtime": 26721.6103,
+    "train_samples": 274422,
+    "train_samples_per_second": 513.483,
+    "train_steps_per_second": 6.42
+}

exp2b/swap_0.7_last_to_push_2128/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.7_last_to_push_2128/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 33.809818097014926,
+    "eval_accuracy": 0.36852503478827553,
+    "eval_loss": 3.566509485244751,
+    "eval_runtime": 81.1363,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 204.927,
+    "eval_steps_per_second": 12.818,
+    "perplexity": 35.39283806929647
+}

exp2b/swap_0.7_last_to_push_2128/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.7_last_to_push_2128/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_push_2128/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_push_2128/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 33.809818097014926,
+    "total_flos": 2.424310847373312e+18,
+    "train_loss": 0.9799438093777361,
+    "train_runtime": 26721.6103,
+    "train_samples": 274422,
+    "train_samples_per_second": 513.483,
+    "train_steps_per_second": 6.42
+}

exp2b/swap_0.7_last_to_push_3591/README.md

@@ -0,0 +1,165 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.7_last_to_push_3591
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/77d49akq)
+# exceptions_exp2_swap_0.7_last_to_push_3591
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5795
+- Accuracy: 0.3663
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 3591
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Validation Loss | Accuracy |
+|:-------------:|:-------:|:------:|:---------------:|:--------:|
+| 4.8245        | 0.2915  | 1000   | 4.7479          | 0.2552   |
+| 4.3478        | 0.5830  | 2000   | 4.2903          | 0.2982   |
+| 4.1604        | 0.8745  | 3000   | 4.1026          | 0.3142   |
+| 3.9932        | 1.1659  | 4000   | 3.9907          | 0.3247   |
+| 3.94          | 1.4574  | 5000   | 3.9161          | 0.3315   |
+| 3.8846        | 1.7489  | 6000   | 3.8589          | 0.3363   |
+| 3.7481        | 2.0402  | 7000   | 3.8169          | 0.3411   |
+| 3.7434        | 2.3317  | 8000   | 3.7852          | 0.3440   |
+| 3.7405        | 2.6233  | 9000   | 3.7572          | 0.3464   |
+| 3.7195        | 2.9148  | 10000  | 3.7314          | 0.3489   |
+| 3.6446        | 3.2061  | 11000  | 3.7183          | 0.3507   |
+| 3.6418        | 3.4976  | 12000  | 3.6975          | 0.3526   |
+| 3.638         | 3.7891  | 13000  | 3.6815          | 0.3546   |
+| 3.5452        | 4.0805  | 14000  | 3.6747          | 0.3553   |
+| 3.5639        | 4.3720  | 15000  | 3.6651          | 0.3565   |
+| 3.5757        | 4.6635  | 16000  | 3.6487          | 0.3579   |
+| 3.5794        | 4.9550  | 17000  | 3.6345          | 0.3592   |
+| 3.4961        | 5.2463  | 18000  | 3.6378          | 0.3595   |
+| 3.5291        | 5.5378  | 19000  | 3.6263          | 0.3604   |
+| 3.5477        | 5.8293  | 20000  | 3.6187          | 0.3614   |
+| 3.4426        | 6.1207  | 21000  | 3.6190          | 0.3619   |
+| 3.4796        | 6.4122  | 22000  | 3.6126          | 0.3624   |
+| 3.5024        | 6.7037  | 23000  | 3.6031          | 0.3634   |
+| 3.496         | 6.9952  | 24000  | 3.5931          | 0.3640   |
+| 3.4205        | 7.2866  | 25000  | 3.6021          | 0.3639   |
+| 3.4508        | 7.5781  | 26000  | 3.5913          | 0.3647   |
+| 3.453         | 7.8696  | 27000  | 3.5832          | 0.3655   |
+| 3.3847        | 8.1609  | 28000  | 3.5938          | 0.3651   |
+| 3.4229        | 8.4524  | 29000  | 3.5880          | 0.3657   |
+| 3.4257        | 8.7439  | 30000  | 3.5795          | 0.3663   |
+| 3.3431        | 9.0353  | 31000  | 3.5823          | 0.3665   |
+| 3.3647        | 9.3268  | 32000  | 3.5805          | 0.3668   |
+| 3.4024        | 9.6183  | 33000  | 3.5725          | 0.3671   |
+| 3.4142        | 9.9098  | 34000  | 3.5659          | 0.3679   |
+| 3.3529        | 10.2011 | 35000  | 3.5776          | 0.3674   |
+| 3.3466        | 10.4927 | 36000  | 3.5697          | 0.3679   |
+| 3.3845        | 10.7842 | 37000  | 3.5609          | 0.3684   |
+| 3.2849        | 11.0755 | 38000  | 3.5759          | 0.3683   |
+| 3.3503        | 11.3670 | 39000  | 3.5705          | 0.3686   |
+| 3.3581        | 11.6585 | 40000  | 3.5622          | 0.3689   |
+| 3.3793        | 11.9500 | 41000  | 3.5508          | 0.3697   |
+| 3.3093        | 12.2414 | 42000  | 3.5660          | 0.3692   |
+| 3.3391        | 12.5329 | 43000  | 3.5611          | 0.3695   |
+| 3.3551        | 12.8244 | 44000  | 3.5527          | 0.3699   |
+| 3.2685        | 13.1157 | 45000  | 3.5655          | 0.3696   |
+| 3.2986        | 13.4072 | 46000  | 3.5590          | 0.3699   |
+| 3.3224        | 13.6988 | 47000  | 3.5538          | 0.3703   |
+| 3.35          | 13.9903 | 48000  | 3.5467          | 0.3709   |
+| 3.2732        | 14.2816 | 49000  | 3.5643          | 0.3701   |
+| 3.3097        | 14.5731 | 50000  | 3.5526          | 0.3708   |
+| 3.322         | 14.8646 | 51000  | 3.5463          | 0.3712   |
+| 3.2602        | 15.1560 | 52000  | 3.5608          | 0.3706   |
+| 3.2819        | 15.4475 | 53000  | 3.5526          | 0.3710   |
+| 3.3111        | 15.7390 | 54000  | 3.5436          | 0.3715   |
+| 3.2119        | 16.0303 | 55000  | 3.5584          | 0.3710   |
+| 3.2562        | 16.3218 | 56000  | 3.5544          | 0.3711   |
+| 3.2765        | 16.6133 | 57000  | 3.5489          | 0.3718   |
+| 3.2937        | 16.9049 | 58000  | 3.5420          | 0.3721   |
+| 3.2387        | 17.1962 | 59000  | 3.5550          | 0.3716   |
+| 3.2742        | 17.4877 | 60000  | 3.5513          | 0.3715   |
+| 3.2854        | 17.7792 | 61000  | 3.5436          | 0.3723   |
+| 3.2008        | 18.0705 | 62000  | 3.5575          | 0.3716   |
+| 3.2348        | 18.3621 | 63000  | 3.5541          | 0.3717   |
+| 3.2599        | 18.6536 | 64000  | 3.5466          | 0.3721   |
+| 3.2804        | 18.9451 | 65000  | 3.5381          | 0.3726   |
+| 3.2044        | 19.2364 | 66000  | 3.5542          | 0.3718   |
+| 3.2455        | 19.5279 | 67000  | 3.5479          | 0.3724   |
+| 3.249         | 19.8194 | 68000  | 3.5413          | 0.3728   |
+| 3.188         | 20.1108 | 69000  | 3.5549          | 0.3718   |
+| 3.2149        | 20.4023 | 70000  | 3.5513          | 0.3721   |
+| 3.2423        | 20.6938 | 71000  | 3.5432          | 0.3729   |
+| 3.2562        | 20.9853 | 72000  | 3.5370          | 0.3734   |
+| 3.1981        | 21.2766 | 73000  | 3.5497          | 0.3724   |
+| 3.224         | 21.5682 | 74000  | 3.5434          | 0.3732   |
+| 3.2341        | 21.8597 | 75000  | 3.5370          | 0.3732   |
+| 3.1725        | 22.1510 | 76000  | 3.5544          | 0.3725   |
+| 3.2035        | 22.4425 | 77000  | 3.5506          | 0.3727   |
+| 3.2282        | 22.7340 | 78000  | 3.5404          | 0.3731   |
+| 3.1312        | 23.0254 | 79000  | 3.5542          | 0.3730   |
+| 3.195         | 23.3169 | 80000  | 3.5524          | 0.3731   |
+| 3.213         | 23.6084 | 81000  | 3.5435          | 0.3735   |
+| 3.2259        | 23.8999 | 82000  | 3.5350          | 0.3739   |
+| 3.1554        | 24.1912 | 83000  | 3.5550          | 0.3727   |
+| 3.1986        | 24.4827 | 84000  | 3.5487          | 0.3733   |
+| 3.2046        | 24.7743 | 85000  | 3.5418          | 0.3737   |
+| 3.1261        | 25.0656 | 86000  | 3.5551          | 0.3732   |
+| 3.1594        | 25.3571 | 87000  | 3.5527          | 0.3731   |
+| 3.1924        | 25.6486 | 88000  | 3.5469          | 0.3735   |
+| 3.1973        | 25.9401 | 89000  | 3.5385          | 0.3740   |
+| 3.1322        | 26.2315 | 90000  | 3.5544          | 0.3730   |
+| 3.1663        | 26.5230 | 91000  | 3.5484          | 0.3738   |
+| 3.1819        | 26.8145 | 92000  | 3.5396          | 0.3743   |
+| 3.1263        | 27.1058 | 93000  | 3.5573          | 0.3734   |
+| 3.1514        | 27.3973 | 94000  | 3.5530          | 0.3733   |
+| 3.1695        | 27.6888 | 95000  | 3.5472          | 0.3739   |
+| 3.1772        | 27.9804 | 96000  | 3.5405          | 0.3744   |
+| 3.144         | 28.2717 | 97000  | 3.5572          | 0.3736   |
+| 3.1562        | 28.5632 | 98000  | 3.5473          | 0.3740   |
+| 3.1756        | 28.8547 | 99000  | 3.5411          | 0.3742   |
+| 3.1147        | 29.1460 | 100000 | 3.5560          | 0.3737   |
+| 3.1334        | 29.4376 | 101000 | 3.5500          | 0.3739   |
+| 3.1493        | 29.7291 | 102000 | 3.5447          | 0.3745   |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.7_last_to_push_3591/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 29.729069496268657,
+    "eval_accuracy": 0.36626454719571133,
+    "eval_loss": 3.5794875621795654,
+    "eval_runtime": 53.5451,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 310.523,
+    "eval_steps_per_second": 19.423,
+    "perplexity": 35.855162597237126,
+    "total_flos": 2.131703487922176e+18,
+    "train_loss": 3.387160031786152,
+    "train_runtime": 45495.4619,
+    "train_samples": 274422,
+    "train_samples_per_second": 301.593,
+    "train_steps_per_second": 3.771
+}

exp2b/swap_0.7_last_to_push_3591/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.7_last_to_push_3591/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 29.729069496268657,
+    "eval_accuracy": 0.36626454719571133,
+    "eval_loss": 3.5794875621795654,
+    "eval_runtime": 53.5451,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 310.523,
+    "eval_steps_per_second": 19.423,
+    "perplexity": 35.855162597237126
+}

exp2b/swap_0.7_last_to_push_3591/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.7_last_to_push_3591/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_push_3591/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_push_3591/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 29.729069496268657,
+    "total_flos": 2.131703487922176e+18,
+    "train_loss": 3.387160031786152,
+    "train_runtime": 45495.4619,
+    "train_samples": 274422,
+    "train_samples_per_second": 301.593,
+    "train_steps_per_second": 3.771
+}

exp2b/swap_0.7_last_to_push_40817/README.md

@@ -0,0 +1,163 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.7_last_to_push_40817
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/hlbt1onb)
+# exceptions_exp2_swap_0.7_last_to_push_40817
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5637
+- Accuracy: 0.3687
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 40817
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Accuracy | Validation Loss |
+|:-------------:|:-------:|:------:|:--------:|:---------------:|
+| 4.8217        | 0.2915  | 1000   | 0.2544   | 4.7554          |
+| 4.3479        | 0.5830  | 2000   | 0.2995   | 4.2855          |
+| 4.1492        | 0.8745  | 3000   | 0.3148   | 4.0985          |
+| 4.0131        | 1.1659  | 4000   | 0.3245   | 3.9932          |
+| 3.941         | 1.4574  | 5000   | 0.3315   | 3.9201          |
+| 3.8662        | 1.7489  | 6000   | 0.3360   | 3.8620          |
+| 3.7463        | 2.0402  | 7000   | 0.3409   | 3.8183          |
+| 3.7449        | 2.3317  | 8000   | 0.3433   | 3.7885          |
+| 3.7373        | 2.6233  | 9000   | 0.3461   | 3.7588          |
+| 3.734         | 2.9148  | 10000  | 0.3487   | 3.7320          |
+| 3.6362        | 3.2061  | 11000  | 0.3506   | 3.7202          |
+| 3.6406        | 3.4976  | 12000  | 0.3522   | 3.7021          |
+| 3.6404        | 3.7891  | 13000  | 0.3539   | 3.6859          |
+| 3.5382        | 4.0805  | 14000  | 0.3551   | 3.6787          |
+| 3.5768        | 4.3720  | 15000  | 0.3563   | 3.6653          |
+| 3.5937        | 4.6635  | 16000  | 0.3573   | 3.6533          |
+| 3.5954        | 4.9550  | 17000  | 0.3588   | 3.6398          |
+| 3.5049        | 5.2463  | 18000  | 0.3592   | 3.6427          |
+| 3.5203        | 5.5378  | 19000  | 0.3603   | 3.6320          |
+| 3.5467        | 5.8293  | 20000  | 0.3610   | 3.6208          |
+| 3.4456        | 6.1207  | 21000  | 0.3615   | 3.6240          |
+| 3.4748        | 6.4122  | 22000  | 0.3620   | 3.6170          |
+| 3.4811        | 6.7037  | 23000  | 0.3627   | 3.6065          |
+| 3.4942        | 6.9952  | 24000  | 0.3637   | 3.5955          |
+| 3.4278        | 7.2866  | 25000  | 0.3637   | 3.6036          |
+| 3.4498        | 7.5781  | 26000  | 0.3641   | 3.5981          |
+| 3.4666        | 7.8696  | 27000  | 0.3649   | 3.5867          |
+| 3.3928        | 8.1609  | 28000  | 0.3648   | 3.5947          |
+| 3.418         | 8.4524  | 29000  | 0.3651   | 3.5900          |
+| 3.4341        | 8.7439  | 30000  | 0.3659   | 3.5826          |
+| 3.3274        | 9.0353  | 31000  | 0.3659   | 3.5890          |
+| 3.3867        | 9.3268  | 32000  | 0.3665   | 3.5828          |
+| 3.4123        | 9.6183  | 33000  | 0.3666   | 3.5777          |
+| 3.4168        | 9.9098  | 34000  | 0.3673   | 3.5678          |
+| 3.3411        | 10.2011 | 35000  | 0.3672   | 3.5789          |
+| 3.3829        | 10.4927 | 36000  | 0.3676   | 3.5734          |
+| 3.3918        | 10.7842 | 37000  | 0.3684   | 3.5650          |
+| 3.2973        | 11.0755 | 38000  | 0.3677   | 3.5801          |
+| 3.3407        | 11.3670 | 39000  | 0.3683   | 3.5712          |
+| 3.3692        | 11.6585 | 40000  | 0.3687   | 3.5637          |
+| 3.377         | 11.9500 | 41000  | 0.3690   | 3.5548          |
+| 3.3323        | 12.2414 | 42000  | 0.3683   | 3.5720          |
+| 3.3416        | 12.5329 | 43000  | 0.3692   | 3.5627          |
+| 3.3447        | 12.8244 | 44000  | 0.3697   | 3.5552          |
+| 3.2705        | 13.1157 | 45000  | 0.3690   | 3.5694          |
+| 3.3022        | 13.4072 | 46000  | 0.3694   | 3.5622          |
+| 3.3421        | 13.6988 | 47000  | 0.3698   | 3.5552          |
+| 3.3492        | 13.9903 | 48000  | 0.3706   | 3.5488          |
+| 3.2805        | 14.2816 | 49000  | 0.3696   | 3.5640          |
+| 3.3032        | 14.5731 | 50000  | 0.3704   | 3.5542          |
+| 3.3201        | 14.8646 | 51000  | 0.3707   | 3.5472          |
+| 3.2533        | 15.1560 | 52000  | 0.3702   | 3.5627          |
+| 3.2841        | 15.4475 | 53000  | 0.3706   | 3.5593          |
+| 3.305         | 15.7390 | 54000  | 0.3711   | 3.5481          |
+| 3.2193        | 16.0303 | 55000  | 0.3709   | 3.5585          |
+| 3.2638        | 16.3218 | 56000  | 0.3706   | 3.5599          |
+| 3.2821        | 16.6133 | 57000  | 0.3713   | 3.5501          |
+| 3.307         | 16.9049 | 58000  | 0.3717   | 3.5459          |
+| 3.2209        | 17.1962 | 59000  | 0.3710   | 3.5593          |
+| 3.2681        | 17.4877 | 60000  | 0.3715   | 3.5543          |
+| 3.3005        | 17.7792 | 61000  | 0.3718   | 3.5449          |
+| 3.1955        | 18.0705 | 62000  | 0.3710   | 3.5593          |
+| 3.244         | 18.3621 | 63000  | 0.3711   | 3.5550          |
+| 3.2692        | 18.6536 | 64000  | 0.3717   | 3.5498          |
+| 3.2766        | 18.9451 | 65000  | 0.3723   | 3.5399          |
+| 3.2099        | 19.2364 | 66000  | 0.3715   | 3.5598          |
+| 3.2416        | 19.5279 | 67000  | 0.3718   | 3.5498          |
+| 3.2588        | 19.8194 | 68000  | 0.3726   | 3.5411          |
+| 3.1787        | 20.1108 | 69000  | 0.3717   | 3.5566          |
+| 3.2107        | 20.4023 | 70000  | 0.3721   | 3.5530          |
+| 3.2439        | 20.6938 | 71000  | 0.3724   | 3.5475          |
+| 3.2457        | 20.9853 | 72000  | 0.3731   | 3.5366          |
+| 3.2009        | 21.2766 | 73000  | 0.3722   | 3.5562          |
+| 3.2171        | 21.5682 | 74000  | 0.3725   | 3.5495          |
+| 3.249         | 21.8597 | 75000  | 0.3730   | 3.5365          |
+| 3.1747        | 22.1510 | 76000  | 0.3722   | 3.5562          |
+| 3.2123        | 22.4425 | 77000  | 0.3721   | 3.5536          |
+| 3.2284        | 22.7340 | 78000  | 0.3729   | 3.5425          |
+| 3.1325        | 23.0254 | 79000  | 0.3726   | 3.5556          |
+| 3.1968        | 23.3169 | 80000  | 0.3722   | 3.5560          |
+| 3.1917        | 23.6084 | 81000  | 3.5561   | 0.3724          |
+| 3.2077        | 23.8999 | 82000  | 3.5492   | 0.3727          |
+| 3.1732        | 24.1915 | 83000  | 3.5601   | 0.3724          |
+| 3.1905        | 24.4830 | 84000  | 3.5547   | 0.3725          |
+| 3.2235        | 24.7745 | 85000  | 3.5428   | 0.3734          |
+| 3.1326        | 25.0659 | 86000  | 3.5563   | 0.3730          |
+| 3.1687        | 25.3574 | 87000  | 3.5527   | 0.3728          |
+| 3.197         | 25.6489 | 88000  | 3.5484   | 0.3731          |
+| 3.2099        | 25.9404 | 89000  | 3.5403   | 0.3737          |
+| 3.1278        | 26.2318 | 90000  | 3.5611   | 0.3730          |
+| 3.1768        | 26.5233 | 91000  | 3.5477   | 0.3734          |
+| 3.1941        | 26.8148 | 92000  | 3.5437   | 0.3737          |
+| 3.1253        | 27.1061 | 93000  | 3.5610   | 0.3725          |
+| 3.1493        | 27.3976 | 94000  | 3.5516   | 0.3732          |
+| 3.1801        | 27.6891 | 95000  | 3.5409   | 0.3739          |
+| 3.1791        | 27.9806 | 96000  | 3.5393   | 0.3741          |
+| 3.1279        | 28.2720 | 97000  | 3.5586   | 0.3730          |
+| 3.1522        | 28.5635 | 98000  | 3.5497   | 0.3735          |
+| 3.1729        | 28.8550 | 99000  | 3.5483   | 0.3739          |
+| 3.1097        | 29.1463 | 100000 | 3.5601   | 0.3730          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.7_last_to_push_40817/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 29.146338619402986,
+    "eval_accuracy": 0.3686502643414614,
+    "eval_loss": 3.5636954307556152,
+    "eval_runtime": 81.2927,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 204.533,
+    "eval_steps_per_second": 12.793,
+    "perplexity": 35.29338069930632,
+    "total_flos": 2.089917666164736e+18,
+    "train_loss": 0.6339143199157715,
+    "train_runtime": 14778.2553,
+    "train_samples": 274422,
+    "train_samples_per_second": 928.465,
+    "train_steps_per_second": 11.608
+}

exp2b/swap_0.7_last_to_push_40817/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.7_last_to_push_40817/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 29.146338619402986,
+    "eval_accuracy": 0.3686502643414614,
+    "eval_loss": 3.5636954307556152,
+    "eval_runtime": 81.2927,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 204.533,
+    "eval_steps_per_second": 12.793,
+    "perplexity": 35.29338069930632
+}

exp2b/swap_0.7_last_to_push_40817/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.7_last_to_push_40817/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.7_last_to_push_40817/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}