Add files using upload-large-folder tool

exp2b/swap_0.3_last_to_carry_40817/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 31.769544686060748,
+    "total_flos": 2.278228220706816e+18,
+    "train_loss": 0.840790353477548,
+    "train_runtime": 21610.366,
+    "train_samples": 274448,
+    "train_samples_per_second": 634.992,
+    "train_steps_per_second": 7.938
+}

exp2b/swap_0.3_last_to_carry_5039/README.md

@@ -0,0 +1,172 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_last_to_carry_5039
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/1mo1375g)
+# exceptions_exp2_swap_0.3_last_to_carry_5039
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5498
+- Accuracy: 0.3741
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 5039
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Accuracy | Validation Loss |
+|:-------------:|:-------:|:------:|:--------:|:---------------:|
+| 4.8376        | 0.2915  | 1000   | 0.2542   | 4.7589          |
+| 4.3508        | 0.5830  | 2000   | 0.2977   | 4.2982          |
+| 4.1599        | 0.8745  | 3000   | 0.3137   | 4.1119          |
+| 4.0058        | 1.1659  | 4000   | 0.3232   | 4.0022          |
+| 3.9397        | 1.4574  | 5000   | 0.3305   | 3.9227          |
+| 3.8862        | 1.7488  | 6000   | 0.3356   | 3.8669          |
+| 3.7487        | 2.0402  | 7000   | 0.3400   | 3.8238          |
+| 3.7661        | 2.3317  | 8000   | 0.3429   | 3.7931          |
+| 3.7449        | 2.6232  | 9000   | 0.3457   | 3.7612          |
+| 3.7305        | 2.9147  | 10000  | 0.3482   | 3.7342          |
+| 3.636         | 3.2061  | 11000  | 0.3503   | 3.7234          |
+| 3.6523        | 3.4976  | 12000  | 0.3518   | 3.7036          |
+| 3.6479        | 3.7891  | 13000  | 0.3535   | 3.6866          |
+| 3.5494        | 4.0805  | 14000  | 0.3549   | 3.6778          |
+| 3.5719        | 4.3719  | 15000  | 0.3561   | 3.6679          |
+| 3.5815        | 4.6634  | 16000  | 0.3575   | 3.6531          |
+| 3.5827        | 4.9549  | 17000  | 0.3585   | 3.6418          |
+| 3.5121        | 5.2463  | 18000  | 0.3592   | 3.6428          |
+| 3.5172        | 5.5378  | 19000  | 0.3597   | 3.6341          |
+| 3.5469        | 5.8293  | 20000  | 0.3607   | 3.6216          |
+| 3.4552        | 6.1207  | 21000  | 0.3614   | 3.6244          |
+| 3.4922        | 6.4122  | 22000  | 0.3619   | 3.6169          |
+| 3.499         | 6.7037  | 23000  | 0.3626   | 3.6087          |
+| 3.4927        | 6.9952  | 24000  | 0.3639   | 3.5962          |
+| 3.4408        | 7.2865  | 25000  | 0.3633   | 3.6067          |
+| 3.4523        | 7.5780  | 26000  | 0.3640   | 3.5957          |
+| 3.4617        | 7.8695  | 27000  | 0.3650   | 3.5857          |
+| 3.396         | 8.1609  | 28000  | 0.3648   | 3.5959          |
+| 3.4334        | 8.4524  | 29000  | 0.3652   | 3.5926          |
+| 3.4364        | 8.7439  | 30000  | 0.3661   | 3.5817          |
+| 3.3397        | 9.0353  | 31000  | 0.3660   | 3.5855          |
+| 3.3905        | 9.3268  | 32000  | 0.3661   | 3.5862          |
+| 3.4009        | 9.6183  | 33000  | 0.3670   | 3.5789          |
+| 3.4185        | 9.9098  | 34000  | 0.3677   | 3.5687          |
+| 3.348         | 10.2011 | 35000  | 0.3672   | 3.5799          |
+| 3.3779        | 10.4926 | 36000  | 0.3676   | 3.5705          |
+| 3.3988        | 10.7841 | 37000  | 0.3681   | 3.5676          |
+| 3.3056        | 11.0755 | 38000  | 0.3679   | 3.5776          |
+| 3.345         | 11.3670 | 39000  | 0.3681   | 3.5718          |
+| 3.3619        | 11.6585 | 40000  | 0.3689   | 3.5643          |
+| 3.387         | 11.9500 | 41000  | 0.3694   | 3.5561          |
+| 3.3217        | 12.2414 | 42000  | 0.3688   | 3.5691          |
+| 3.3443        | 12.5329 | 43000  | 0.3691   | 3.5631          |
+| 3.3572        | 12.8243 | 44000  | 0.3696   | 3.5544          |
+| 3.282         | 13.1157 | 45000  | 0.3690   | 3.5689          |
+| 3.3217        | 13.4072 | 46000  | 0.3696   | 3.5634          |
+| 3.333         | 13.6987 | 47000  | 0.3703   | 3.5528          |
+| 3.3499        | 13.9902 | 48000  | 0.3704   | 3.5476          |
+| 3.2773        | 14.2816 | 49000  | 0.3698   | 3.5656          |
+| 3.3086        | 14.5731 | 50000  | 0.3703   | 3.5580          |
+| 3.343         | 14.8646 | 51000  | 0.3706   | 3.5467          |
+| 3.2598        | 15.1559 | 52000  | 0.3702   | 3.5646          |
+| 3.2774        | 15.4474 | 53000  | 0.3705   | 3.5598          |
+| 3.3126        | 15.7389 | 54000  | 0.3711   | 3.5496          |
+| 3.2012        | 16.0303 | 55000  | 0.3705   | 3.5610          |
+| 3.2648        | 16.3218 | 56000  | 0.3711   | 3.5536          |
+| 3.2782        | 16.6133 | 57000  | 0.3710   | 3.5503          |
+| 3.3024        | 16.9048 | 58000  | 0.3720   | 3.5396          |
+| 3.2318        | 17.1962 | 59000  | 0.3710   | 3.5579          |
+| 3.2653        | 17.4877 | 60000  | 0.3715   | 3.5493          |
+| 3.274         | 17.7792 | 61000  | 0.3722   | 3.5431          |
+| 3.1885        | 18.0705 | 62000  | 0.3713   | 3.5596          |
+| 3.2372        | 18.3620 | 63000  | 0.3717   | 3.5528          |
+| 3.2585        | 18.6535 | 64000  | 0.3721   | 3.5450          |
+| 3.279         | 18.9450 | 65000  | 0.3726   | 3.5389          |
+| 3.2051        | 19.2364 | 66000  | 0.3718   | 3.5559          |
+| 3.2422        | 19.5279 | 67000  | 0.3718   | 3.5507          |
+| 3.2517        | 19.8194 | 68000  | 0.3726   | 3.5421          |
+| 3.1962        | 20.1108 | 69000  | 0.3719   | 3.5548          |
+| 3.2241        | 20.4023 | 70000  | 0.3721   | 3.5511          |
+| 3.2393        | 20.6938 | 71000  | 0.3724   | 3.5465          |
+| 3.2653        | 20.9853 | 72000  | 0.3729   | 3.5397          |
+| 3.2169        | 21.2766 | 73000  | 0.3719   | 3.5543          |
+| 3.2173        | 21.5681 | 74000  | 0.3726   | 3.5482          |
+| 3.2412        | 21.8596 | 75000  | 0.3729   | 3.5411          |
+| 3.1695        | 22.1510 | 76000  | 0.3725   | 3.5561          |
+| 3.2119        | 22.4425 | 77000  | 0.3726   | 3.5493          |
+| 3.2316        | 22.7340 | 78000  | 0.3731   | 3.5434          |
+| 3.1426        | 23.0254 | 79000  | 0.3723   | 3.5559          |
+| 3.1847        | 23.3169 | 80000  | 0.3729   | 3.5535          |
+| 3.1668        | 23.6083 | 81000  | 3.5551   | 0.3725          |
+| 3.2001        | 23.8998 | 82000  | 3.5479   | 0.3731          |
+| 3.1768        | 24.1915 | 83000  | 3.5602   | 0.3723          |
+| 3.1871        | 24.4830 | 84000  | 3.5519   | 0.3727          |
+| 3.2136        | 24.7745 | 85000  | 3.5428   | 0.3733          |
+| 3.1267        | 25.0659 | 86000  | 3.5576   | 0.3731          |
+| 3.1684        | 25.3574 | 87000  | 3.5551   | 0.3727          |
+| 3.2026        | 25.6489 | 88000  | 3.5472   | 0.3735          |
+| 3.2202        | 25.9404 | 89000  | 3.5379   | 0.3737          |
+| 3.1603        | 26.2317 | 90000  | 3.5586   | 0.3729          |
+| 3.1896        | 26.5232 | 91000  | 3.5494   | 0.3735          |
+| 3.1884        | 26.8147 | 92000  | 3.5417   | 0.3740          |
+| 3.1285        | 27.1061 | 93000  | 3.5546   | 0.3731          |
+| 3.1577        | 27.3976 | 94000  | 3.5499   | 0.3736          |
+| 3.183         | 27.6891 | 95000  | 3.5481   | 0.3736          |
+| 3.1974        | 27.9806 | 96000  | 3.5389   | 0.3741          |
+| 3.1403        | 28.2720 | 97000  | 3.5531   | 0.3733          |
+| 3.1545        | 28.5635 | 98000  | 3.5498   | 0.3738          |
+| 3.165         | 28.8550 | 99000  | 3.5403   | 0.3740          |
+| 3.119         | 29.1463 | 100000 | 3.5620   | 0.3732          |
+| 3.1416        | 29.4378 | 101000 | 3.5507   | 0.3740          |
+| 3.1652        | 29.7293 | 102000 | 3.5452   | 0.3741          |
+| 3.0738        | 30.0207 | 103000 | 3.5542   | 0.3740          |
+| 3.1227        | 30.3122 | 104000 | 3.5568   | 0.3737          |
+| 3.1518        | 30.6037 | 105000 | 3.5512   | 0.3741          |
+| 3.1593        | 30.8952 | 106000 | 3.5423   | 0.3742          |
+| 3.1051        | 31.1866 | 107000 | 3.5578   | 0.3737          |
+| 3.1409        | 31.4781 | 108000 | 3.5499   | 0.3742          |
+| 3.1566        | 31.7695 | 109000 | 3.5498   | 0.3741          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_last_to_carry_5039/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 31.769544686060748,
+    "eval_accuracy": 0.3741262325665809,
+    "eval_loss": 3.5498111248016357,
+    "eval_runtime": 80.158,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 207.428,
+    "eval_steps_per_second": 12.974,
+    "perplexity": 34.806742736279105,
+    "total_flos": 2.278228220706816e+18,
+    "train_loss": 0.8412648971977584,
+    "train_runtime": 21257.5406,
+    "train_samples": 274448,
+    "train_samples_per_second": 645.531,
+    "train_steps_per_second": 8.07
+}

exp2b/swap_0.3_last_to_carry_5039/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_last_to_carry_5039/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 31.769544686060748,
+    "eval_accuracy": 0.3741262325665809,
+    "eval_loss": 3.5498111248016357,
+    "eval_runtime": 80.158,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 207.428,
+    "eval_steps_per_second": 12.974,
+    "perplexity": 34.806742736279105
+}

exp2b/swap_0.3_last_to_carry_5039/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_last_to_carry_5039/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_carry_5039/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_carry_5039/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 31.769544686060748,
+    "total_flos": 2.278228220706816e+18,
+    "train_loss": 0.8412648971977584,
+    "train_runtime": 21257.5406,
+    "train_samples": 274448,
+    "train_samples_per_second": 645.531,
+    "train_steps_per_second": 8.07
+}

exp2b/swap_0.3_last_to_drop_1032/README.md

@@ -0,0 +1,172 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_last_to_drop_1032
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/r8hhxvgj)
+# exceptions_exp2_swap_0.3_last_to_drop_1032
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5508
+- Accuracy: 0.3741
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 1032
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Accuracy | Validation Loss |
+|:-------------:|:-------:|:------:|:--------:|:---------------:|
+| 4.8235        | 0.2915  | 1000   | 0.2544   | 4.7551          |
+| 4.3553        | 0.5830  | 2000   | 0.2984   | 4.2897          |
+| 4.1484        | 0.8745  | 3000   | 0.3139   | 4.1042          |
+| 4.0041        | 1.1659  | 4000   | 0.3239   | 3.9982          |
+| 3.9468        | 1.4574  | 5000   | 0.3306   | 3.9197          |
+| 3.8875        | 1.7488  | 6000   | 0.3360   | 3.8670          |
+| 3.7501        | 2.0402  | 7000   | 0.3403   | 3.8202          |
+| 3.7497        | 2.3317  | 8000   | 0.3434   | 3.7905          |
+| 3.7424        | 2.6232  | 9000   | 0.3458   | 3.7606          |
+| 3.729         | 2.9147  | 10000  | 0.3485   | 3.7343          |
+| 3.6452        | 3.2061  | 11000  | 0.3502   | 3.7250          |
+| 3.6466        | 3.4976  | 12000  | 0.3519   | 3.7026          |
+| 3.6575        | 3.7891  | 13000  | 0.3535   | 3.6850          |
+| 3.5567        | 4.0805  | 14000  | 0.3548   | 3.6800          |
+| 3.5675        | 4.3719  | 15000  | 0.3561   | 3.6665          |
+| 3.5869        | 4.6634  | 16000  | 0.3568   | 3.6547          |
+| 3.5789        | 4.9549  | 17000  | 0.3582   | 3.6421          |
+| 3.5123        | 5.2463  | 18000  | 0.3591   | 3.6440          |
+| 3.512         | 5.5378  | 19000  | 0.3600   | 3.6330          |
+| 3.5442        | 5.8293  | 20000  | 0.3609   | 3.6218          |
+| 3.4481        | 6.1207  | 21000  | 0.3610   | 3.6227          |
+| 3.4852        | 6.4122  | 22000  | 0.3617   | 3.6196          |
+| 3.4867        | 6.7037  | 23000  | 0.3629   | 3.6085          |
+| 3.5079        | 6.9952  | 24000  | 0.3632   | 3.6005          |
+| 3.4287        | 7.2865  | 25000  | 0.3628   | 3.6080          |
+| 3.4631        | 7.5780  | 26000  | 0.3641   | 3.5972          |
+| 3.468         | 7.8695  | 27000  | 0.3646   | 3.5895          |
+| 3.3731        | 8.1609  | 28000  | 0.3646   | 3.6000          |
+| 3.4144        | 8.4524  | 29000  | 0.3654   | 3.5911          |
+| 3.4449        | 8.7439  | 30000  | 0.3654   | 3.5835          |
+| 3.3304        | 9.0353  | 31000  | 0.3658   | 3.5865          |
+| 3.3887        | 9.3268  | 32000  | 0.3660   | 3.5869          |
+| 3.3909        | 9.6183  | 33000  | 0.3667   | 3.5774          |
+| 3.4112        | 9.9098  | 34000  | 0.3672   | 3.5700          |
+| 3.3481        | 10.2011 | 35000  | 0.3669   | 3.5828          |
+| 3.3647        | 10.4926 | 36000  | 0.3677   | 3.5744          |
+| 3.3856        | 10.7841 | 37000  | 0.3680   | 3.5664          |
+| 3.3035        | 11.0755 | 38000  | 0.3679   | 3.5784          |
+| 3.345         | 11.3670 | 39000  | 0.3680   | 3.5718          |
+| 3.3675        | 11.6585 | 40000  | 0.3688   | 3.5642          |
+| 3.3699        | 11.9500 | 41000  | 0.3690   | 3.5599          |
+| 3.3244        | 12.2414 | 42000  | 0.3686   | 3.5710          |
+| 3.3428        | 12.5329 | 43000  | 0.3691   | 3.5675          |
+| 3.3582        | 12.8243 | 44000  | 0.3697   | 3.5583          |
+| 3.281         | 13.1157 | 45000  | 0.3687   | 3.5718          |
+| 3.3096        | 13.4072 | 46000  | 0.3696   | 3.5636          |
+| 3.3312        | 13.6987 | 47000  | 0.3698   | 3.5557          |
+| 3.3427        | 13.9902 | 48000  | 0.3706   | 3.5487          |
+| 3.2845        | 14.2816 | 49000  | 0.3697   | 3.5665          |
+| 3.3226        | 14.5731 | 50000  | 0.3702   | 3.5575          |
+| 3.3355        | 14.8646 | 51000  | 0.3706   | 3.5465          |
+| 3.258         | 15.1559 | 52000  | 0.3701   | 3.5647          |
+| 3.2921        | 15.4474 | 53000  | 0.3703   | 3.5605          |
+| 3.3033        | 15.7389 | 54000  | 0.3708   | 3.5486          |
+| 3.2211        | 16.0303 | 55000  | 0.3706   | 3.5614          |
+| 3.2648        | 16.3218 | 56000  | 0.3708   | 3.5589          |
+| 3.2845        | 16.6133 | 57000  | 0.3710   | 3.5573          |
+| 3.3076        | 16.9048 | 58000  | 0.3716   | 3.5420          |
+| 3.2284        | 17.1962 | 59000  | 0.3708   | 3.5640          |
+| 3.2602        | 17.4877 | 60000  | 0.3710   | 3.5527          |
+| 3.2906        | 17.7792 | 61000  | 0.3720   | 3.5479          |
+| 3.2018        | 18.0705 | 62000  | 0.3713   | 3.5612          |
+| 3.2406        | 18.3620 | 63000  | 0.3714   | 3.5581          |
+| 3.2617        | 18.6535 | 64000  | 0.3720   | 3.5489          |
+| 3.2833        | 18.9450 | 65000  | 0.3723   | 3.5409          |
+| 3.2291        | 19.2364 | 66000  | 0.3710   | 3.5608          |
+| 3.2556        | 19.5279 | 67000  | 0.3718   | 3.5511          |
+| 3.2603        | 19.8194 | 68000  | 0.3723   | 3.5435          |
+| 3.1786        | 20.1108 | 69000  | 0.3714   | 3.5614          |
+| 3.2258        | 20.4023 | 70000  | 0.3718   | 3.5557          |
+| 3.2437        | 20.6938 | 71000  | 0.3724   | 3.5454          |
+| 3.2571        | 20.9853 | 72000  | 0.3726   | 3.5407          |
+| 3.2064        | 21.2766 | 73000  | 0.3720   | 3.5581          |
+| 3.2243        | 21.5681 | 74000  | 0.3722   | 3.5497          |
+| 3.247         | 21.8596 | 75000  | 0.3732   | 3.5432          |
+| 3.1678        | 22.1510 | 76000  | 0.3722   | 3.5580          |
+| 3.2098        | 22.4425 | 77000  | 0.3727   | 3.5515          |
+| 3.2263        | 22.7340 | 78000  | 0.3727   | 3.5462          |
+| 3.137         | 23.0254 | 79000  | 0.3725   | 3.5595          |
+| 3.1979        | 23.3169 | 80000  | 0.3724   | 3.5566          |
+| 3.1956        | 23.6083 | 81000  | 3.5573   | 0.3724          |
+| 3.2112        | 23.8998 | 82000  | 3.5530   | 0.3726          |
+| 3.1659        | 24.1915 | 83000  | 3.5616   | 0.3724          |
+| 3.1969        | 24.4830 | 84000  | 3.5556   | 0.3726          |
+| 3.2202        | 24.7745 | 85000  | 3.5448   | 0.3730          |
+| 3.1451        | 25.0659 | 86000  | 3.5617   | 0.3724          |
+| 3.1759        | 25.3574 | 87000  | 3.5573   | 0.3726          |
+| 3.1946        | 25.6489 | 88000  | 3.5505   | 0.3730          |
+| 3.2097        | 25.9404 | 89000  | 3.5389   | 0.3739          |
+| 3.1585        | 26.2317 | 90000  | 3.5570   | 0.3728          |
+| 3.1781        | 26.5232 | 91000  | 3.5499   | 0.3732          |
+| 3.2103        | 26.8147 | 92000  | 3.5439   | 0.3736          |
+| 3.121         | 27.1061 | 93000  | 3.5603   | 0.3727          |
+| 3.1601        | 27.3976 | 94000  | 3.5555   | 0.3731          |
+| 3.1809        | 27.6891 | 95000  | 3.5464   | 0.3735          |
+| 3.1921        | 27.9806 | 96000  | 3.5419   | 0.3740          |
+| 3.1305        | 28.2720 | 97000  | 3.5572   | 0.3730          |
+| 3.1726        | 28.5635 | 98000  | 3.5512   | 0.3734          |
+| 3.1832        | 28.8550 | 99000  | 3.5419   | 0.3743          |
+| 3.1176        | 29.1463 | 100000 | 3.5623   | 0.3733          |
+| 3.1431        | 29.4378 | 101000 | 3.5572   | 0.3734          |
+| 3.1734        | 29.7293 | 102000 | 3.5512   | 0.3736          |
+| 3.0923        | 30.0207 | 103000 | 3.5627   | 0.3732          |
+| 3.1347        | 30.3122 | 104000 | 3.5604   | 0.3733          |
+| 3.1553        | 30.6037 | 105000 | 3.5488   | 0.3742          |
+| 3.1699        | 30.8952 | 106000 | 3.5426   | 0.3745          |
+| 3.1223        | 31.1866 | 107000 | 3.5599   | 0.3734          |
+| 3.1372        | 31.4781 | 108000 | 3.5551   | 0.3739          |
+| 3.153         | 31.7695 | 109000 | 3.5508   | 0.3741          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_last_to_drop_1032/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 31.769544686060748,
+    "eval_accuracy": 0.3740895111186542,
+    "eval_loss": 3.5507755279541016,
+    "eval_runtime": 83.7685,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 198.488,
+    "eval_steps_per_second": 12.415,
+    "perplexity": 34.84032666031919,
+    "total_flos": 2.278228220706816e+18,
+    "train_loss": 0.8417494567031161,
+    "train_runtime": 21888.5243,
+    "train_samples": 274448,
+    "train_samples_per_second": 626.922,
+    "train_steps_per_second": 7.837
+}

exp2b/swap_0.3_last_to_drop_1032/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_last_to_drop_1032/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 31.769544686060748,
+    "eval_accuracy": 0.3740895111186542,
+    "eval_loss": 3.5507755279541016,
+    "eval_runtime": 83.7685,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 198.488,
+    "eval_steps_per_second": 12.415,
+    "perplexity": 34.84032666031919
+}

exp2b/swap_0.3_last_to_drop_1032/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_last_to_drop_1032/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_drop_1032/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_drop_1032/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 31.769544686060748,
+    "total_flos": 2.278228220706816e+18,
+    "train_loss": 0.8417494567031161,
+    "train_runtime": 21888.5243,
+    "train_samples": 274448,
+    "train_samples_per_second": 626.922,
+    "train_steps_per_second": 7.837
+}

exp2b/swap_0.3_last_to_drop_2128/README.md

@@ -0,0 +1,155 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_last_to_drop_2128
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/eae5rox2)
+# exceptions_exp2_swap_0.3_last_to_drop_2128
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5646
+- Accuracy: 0.3686
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 2128
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Validation Loss | Accuracy |
+|:-------------:|:-------:|:-----:|:---------------:|:--------:|
+| 4.8398        | 0.2915  | 1000  | 4.7639          | 0.2529   |
+| 4.3477        | 0.5830  | 2000  | 4.2884          | 0.2984   |
+| 4.1512        | 0.8745  | 3000  | 4.1016          | 0.3147   |
+| 4.0025        | 1.1659  | 4000  | 3.9974          | 0.3241   |
+| 3.9406        | 1.4574  | 5000  | 3.9240          | 0.3308   |
+| 3.8856        | 1.7488  | 6000  | 3.8649          | 0.3357   |
+| 3.7587        | 2.0402  | 7000  | 3.8219          | 0.3400   |
+| 3.7482        | 2.3317  | 8000  | 3.7934          | 0.3431   |
+| 3.7419        | 2.6232  | 9000  | 3.7618          | 0.3457   |
+| 3.7263        | 2.9147  | 10000 | 3.7352          | 0.3481   |
+| 3.6361        | 3.2061  | 11000 | 3.7234          | 0.3502   |
+| 3.6494        | 3.4976  | 12000 | 3.7038          | 0.3519   |
+| 3.6575        | 3.7891  | 13000 | 3.6854          | 0.3537   |
+| 3.5491        | 4.0805  | 14000 | 3.6794          | 0.3548   |
+| 3.5768        | 4.3719  | 15000 | 3.6669          | 0.3560   |
+| 3.578         | 4.6634  | 16000 | 3.6552          | 0.3572   |
+| 3.5855        | 4.9549  | 17000 | 3.6396          | 0.3585   |
+| 3.5118        | 5.2463  | 18000 | 3.6447          | 0.3589   |
+| 3.5352        | 5.5378  | 19000 | 3.6325          | 0.3596   |
+| 3.5396        | 5.8293  | 20000 | 3.6209          | 0.3609   |
+| 3.4371        | 6.1207  | 21000 | 3.6238          | 0.3609   |
+| 3.4827        | 6.4122  | 22000 | 3.6158          | 0.3620   |
+| 3.5052        | 6.7037  | 23000 | 3.6065          | 0.3628   |
+| 3.5018        | 6.9952  | 24000 | 3.5982          | 0.3633   |
+| 3.4366        | 7.2865  | 25000 | 3.6080          | 0.3634   |
+| 3.4482        | 7.5780  | 26000 | 3.5993          | 0.3641   |
+| 3.477         | 7.8695  | 27000 | 3.5876          | 0.3646   |
+| 3.3894        | 8.1609  | 28000 | 3.5967          | 0.3648   |
+| 3.4054        | 8.4524  | 29000 | 3.5929          | 0.3653   |
+| 3.4432        | 8.7439  | 30000 | 3.5822          | 0.3660   |
+| 3.3254        | 9.0353  | 31000 | 3.5891          | 0.3657   |
+| 3.3811        | 9.3268  | 32000 | 3.5864          | 0.3661   |
+| 3.4141        | 9.6183  | 33000 | 3.5767          | 0.3667   |
+| 3.4197        | 9.9098  | 34000 | 3.5686          | 0.3675   |
+| 3.3474        | 10.2011 | 35000 | 3.5831          | 0.3671   |
+| 3.3755        | 10.4926 | 36000 | 3.5738          | 0.3676   |
+| 3.3875        | 10.7841 | 37000 | 3.5648          | 0.3679   |
+| 3.2944        | 11.0755 | 38000 | 3.5789          | 0.3675   |
+| 3.3387        | 11.3670 | 39000 | 3.5726          | 0.3680   |
+| 3.3671        | 11.6585 | 40000 | 3.5646          | 0.3686   |
+| 3.3927        | 11.9500 | 41000 | 3.5561          | 0.3692   |
+| 3.3026        | 12.2414 | 42000 | 3.5746          | 0.3683   |
+| 3.3385        | 12.5329 | 43000 | 3.5650          | 0.3690   |
+| 3.3527        | 12.8243 | 44000 | 3.5567          | 0.3693   |
+| 3.2774        | 13.1157 | 45000 | 3.5703          | 0.3691   |
+| 3.3081        | 13.4072 | 46000 | 3.5631          | 0.3696   |
+| 3.3281        | 13.6987 | 47000 | 3.5540          | 0.3700   |
+| 3.3524        | 13.9902 | 48000 | 3.5492          | 0.3705   |
+| 3.2832        | 14.2816 | 49000 | 3.5661          | 0.3697   |
+| 3.3161        | 14.5731 | 50000 | 3.5575          | 0.3702   |
+| 3.3222        | 14.8646 | 51000 | 3.5501          | 0.3704   |
+| 3.2407        | 15.1559 | 52000 | 3.5644          | 0.3703   |
+| 3.295         | 15.4474 | 53000 | 3.5573          | 0.3704   |
+| 3.3097        | 15.7389 | 54000 | 3.5522          | 0.3711   |
+| 3.2056        | 16.0303 | 55000 | 3.5602          | 0.3707   |
+| 3.2651        | 16.3218 | 56000 | 3.5586          | 0.3706   |
+| 3.2933        | 16.6133 | 57000 | 3.5517          | 0.3713   |
+| 3.3089        | 16.9048 | 58000 | 3.5439          | 0.3719   |
+| 3.2379        | 17.1962 | 59000 | 3.5605          | 0.3710   |
+| 3.2779        | 17.4877 | 60000 | 3.5536          | 0.3714   |
+| 3.284         | 17.7792 | 61000 | 3.5476          | 0.3718   |
+| 3.2035        | 18.0705 | 62000 | 3.5592          | 0.3715   |
+| 3.2347        | 18.3620 | 63000 | 3.5548          | 0.3715   |
+| 3.2669        | 18.6535 | 64000 | 3.5478          | 0.3720   |
+| 3.2758        | 18.9450 | 65000 | 3.5397          | 0.3725   |
+| 3.2212        | 19.2364 | 66000 | 3.5577          | 0.3716   |
+| 3.2378        | 19.5279 | 67000 | 3.5522          | 0.3718   |
+| 3.259         | 19.8194 | 68000 | 3.5416          | 0.3726   |
+| 3.1829        | 20.1108 | 69000 | 3.5602          | 0.3717   |
+| 3.2376        | 20.4023 | 70000 | 3.5571          | 0.3719   |
+| 3.2464        | 20.6938 | 71000 | 3.5437          | 0.3726   |
+| 3.2623        | 20.9853 | 72000 | 3.5346          | 0.3733   |
+| 3.1918        | 21.2766 | 73000 | 3.5591          | 0.3722   |
+| 3.2393        | 21.5681 | 74000 | 3.5503          | 0.3728   |
+| 3.2322        | 21.8596 | 75000 | 3.5422          | 0.3731   |
+| 3.1831        | 22.1510 | 76000 | 3.5584          | 0.3723   |
+| 3.2067        | 22.4425 | 77000 | 3.5529          | 0.3727   |
+| 3.2365        | 22.7340 | 78000 | 3.5416          | 0.3733   |
+| 3.1465        | 23.0254 | 79000 | 3.5557          | 0.3727   |
+| 3.19          | 23.3169 | 80000 | 3.5559          | 0.3722   |
+| 3.2033        | 23.6083 | 81000 | 3.5450          | 0.3731   |
+| 3.2302        | 23.8998 | 82000 | 3.5389          | 0.3736   |
+| 3.1642        | 24.1912 | 83000 | 3.5607          | 0.3723   |
+| 3.1933        | 24.4827 | 84000 | 3.5481          | 0.3730   |
+| 3.2109        | 24.7742 | 85000 | 3.5413          | 0.3734   |
+| 3.1286        | 25.0656 | 86000 | 3.5565          | 0.3728   |
+| 3.1795        | 25.3571 | 87000 | 3.5517          | 0.3732   |
+| 3.1963        | 25.6486 | 88000 | 3.5447          | 0.3736   |
+| 3.2196        | 25.9401 | 89000 | 3.5381          | 0.3738   |
+| 3.1568        | 26.2314 | 90000 | 3.5570          | 0.3729   |
+| 3.1805        | 26.5229 | 91000 | 3.5485          | 0.3735   |
+| 3.2022        | 26.8144 | 92000 | 3.5413          | 0.3738   |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_last_to_drop_2128/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 26.814434792747626,
+    "eval_accuracy": 0.3685549298131902,
+    "eval_loss": 3.5645804405212402,
+    "eval_runtime": 89.3926,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 186.0,
+    "eval_steps_per_second": 11.634,
+    "perplexity": 35.32462951159953,
+    "total_flos": 1.922891960549376e+18,
+    "train_loss": 3.417767117375913,
+    "train_runtime": 69305.2293,
+    "train_samples": 274448,
+    "train_samples_per_second": 197.999,
+    "train_steps_per_second": 2.475
+}

exp2b/swap_0.3_last_to_drop_2128/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_last_to_drop_2128/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 26.814434792747626,
+    "eval_accuracy": 0.3685549298131902,
+    "eval_loss": 3.5645804405212402,
+    "eval_runtime": 89.3926,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 186.0,
+    "eval_steps_per_second": 11.634,
+    "perplexity": 35.32462951159953
+}

exp2b/swap_0.3_last_to_drop_2128/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_last_to_drop_2128/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_drop_2128/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_drop_2128/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 26.814434792747626,
+    "total_flos": 1.922891960549376e+18,
+    "train_loss": 3.417767117375913,
+    "train_runtime": 69305.2293,
+    "train_samples": 274448,
+    "train_samples_per_second": 197.999,
+    "train_steps_per_second": 2.475
+}

exp2b/swap_0.3_last_to_drop_3591/README.md

@@ -0,0 +1,158 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_last_to_drop_3591
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/q4a2ocff)
+# exceptions_exp2_swap_0.3_last_to_drop_3591
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5611
+- Accuracy: 0.3692
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 3591
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Validation Loss | Accuracy |
+|:-------------:|:-------:|:-----:|:---------------:|:--------:|
+| 4.8307        | 0.2915  | 1000  | 4.7528          | 0.2547   |
+| 4.3399        | 0.5830  | 2000  | 4.2888          | 0.2988   |
+| 4.1412        | 0.8745  | 3000  | 4.0976          | 0.3150   |
+| 3.999         | 1.1659  | 4000  | 3.9911          | 0.3247   |
+| 3.9366        | 1.4574  | 5000  | 3.9156          | 0.3315   |
+| 3.8932        | 1.7488  | 6000  | 3.8589          | 0.3366   |
+| 3.7472        | 2.0402  | 7000  | 3.8156          | 0.3406   |
+| 3.7682        | 2.3317  | 8000  | 3.7874          | 0.3440   |
+| 3.7343        | 2.6232  | 9000  | 3.7559          | 0.3465   |
+| 3.7197        | 2.9147  | 10000 | 3.7309          | 0.3487   |
+| 3.6492        | 3.2061  | 11000 | 3.7177          | 0.3511   |
+| 3.6586        | 3.4976  | 12000 | 3.6991          | 0.3525   |
+| 3.6414        | 3.7891  | 13000 | 3.6815          | 0.3543   |
+| 3.5561        | 4.0805  | 14000 | 3.6759          | 0.3554   |
+| 3.577         | 4.3719  | 15000 | 3.6633          | 0.3563   |
+| 3.5849        | 4.6634  | 16000 | 3.6510          | 0.3577   |
+| 3.587         | 4.9549  | 17000 | 3.6385          | 0.3591   |
+| 3.509         | 5.2463  | 18000 | 3.6400          | 0.3596   |
+| 3.5213        | 5.5378  | 19000 | 3.6289          | 0.3604   |
+| 3.5433        | 5.8293  | 20000 | 3.6175          | 0.3614   |
+| 3.4371        | 6.1207  | 21000 | 3.6213          | 0.3615   |
+| 3.4752        | 6.4122  | 22000 | 3.6150          | 0.3622   |
+| 3.4955        | 6.7037  | 23000 | 3.6015          | 0.3633   |
+| 3.5007        | 6.9952  | 24000 | 3.5953          | 0.3639   |
+| 3.4315        | 7.2865  | 25000 | 3.6035          | 0.3638   |
+| 3.4545        | 7.5780  | 26000 | 3.5948          | 0.3646   |
+| 3.4606        | 7.8695  | 27000 | 3.5853          | 0.3654   |
+| 3.3771        | 8.1609  | 28000 | 3.5933          | 0.3651   |
+| 3.412         | 8.4524  | 29000 | 3.5879          | 0.3655   |
+| 3.4258        | 8.7439  | 30000 | 3.5777          | 0.3663   |
+| 3.3274        | 9.0353  | 31000 | 3.5839          | 0.3664   |
+| 3.3857        | 9.3268  | 32000 | 3.5825          | 0.3668   |
+| 3.4067        | 9.6183  | 33000 | 3.5739          | 0.3672   |
+| 3.4288        | 9.9098  | 34000 | 3.5648          | 0.3679   |
+| 3.3344        | 10.2011 | 35000 | 3.5775          | 0.3675   |
+| 3.3684        | 10.4926 | 36000 | 3.5698          | 0.3681   |
+| 3.394         | 10.7841 | 37000 | 3.5625          | 0.3685   |
+| 3.2872        | 11.0755 | 38000 | 3.5743          | 0.3684   |
+| 3.3392        | 11.3670 | 39000 | 3.5675          | 0.3683   |
+| 3.3675        | 11.6585 | 40000 | 3.5611          | 0.3692   |
+| 3.377         | 11.9500 | 41000 | 3.5557          | 0.3693   |
+| 3.3208        | 12.2414 | 42000 | 3.5674          | 0.3687   |
+| 3.3352        | 12.5329 | 43000 | 3.5571          | 0.3696   |
+| 3.3566        | 12.8243 | 44000 | 3.5539          | 0.3703   |
+| 3.2877        | 13.1157 | 45000 | 3.5664          | 0.3696   |
+| 3.304         | 13.4072 | 46000 | 3.5602          | 0.3700   |
+| 3.3343        | 13.6987 | 47000 | 3.5514          | 0.3703   |
+| 3.348         | 13.9902 | 48000 | 3.5433          | 0.3707   |
+| 3.2943        | 14.2816 | 49000 | 3.5629          | 0.3701   |
+| 3.309         | 14.5731 | 50000 | 3.5542          | 0.3708   |
+| 3.3244        | 14.8646 | 51000 | 3.5442          | 0.3712   |
+| 3.2472        | 15.1559 | 52000 | 3.5590          | 0.3708   |
+| 3.2813        | 15.4474 | 53000 | 3.5533          | 0.3710   |
+| 3.2968        | 15.7389 | 54000 | 3.5465          | 0.3716   |
+| 3.2181        | 16.0303 | 55000 | 3.5576          | 0.3709   |
+| 3.2577        | 16.3218 | 56000 | 3.5567          | 0.3710   |
+| 3.2813        | 16.6133 | 57000 | 3.5463          | 0.3716   |
+| 3.3009        | 16.9048 | 58000 | 3.5392          | 0.3722   |
+| 3.2308        | 17.1962 | 59000 | 3.5570          | 0.3713   |
+| 3.2594        | 17.4877 | 60000 | 3.5497          | 0.3717   |
+| 3.279         | 17.7792 | 61000 | 3.5440          | 0.3720   |
+| 3.2003        | 18.0705 | 62000 | 3.5573          | 0.3714   |
+| 3.2396        | 18.3620 | 63000 | 3.5517          | 0.3719   |
+| 3.2702        | 18.6535 | 64000 | 3.5449          | 0.3723   |
+| 3.2764        | 18.9450 | 65000 | 3.5358          | 0.3728   |
+| 3.217         | 19.2364 | 66000 | 3.5490          | 0.3719   |
+| 3.2504        | 19.5279 | 67000 | 3.5482          | 0.3722   |
+| 3.2675        | 19.8194 | 68000 | 3.5371          | 0.3730   |
+| 3.1747        | 20.1108 | 69000 | 3.5570          | 0.3720   |
+| 3.2261        | 20.4023 | 70000 | 3.5480          | 0.3729   |
+| 3.2419        | 20.6938 | 71000 | 3.5406          | 0.3730   |
+| 3.2373        | 20.9853 | 72000 | 3.5357          | 0.3732   |
+| 3.2006        | 21.2766 | 73000 | 3.5510          | 0.3724   |
+| 3.2397        | 21.5681 | 74000 | 3.5452          | 0.3728   |
+| 3.2436        | 21.8596 | 75000 | 3.5327          | 0.3735   |
+| 3.1737        | 22.1510 | 76000 | 3.5524          | 0.3729   |
+| 3.2065        | 22.4425 | 77000 | 3.5469          | 0.3728   |
+| 3.2299        | 22.7340 | 78000 | 3.5368          | 0.3734   |
+| 3.1314        | 23.0254 | 79000 | 3.5550          | 0.3730   |
+| 3.1791        | 23.3169 | 80000 | 3.5532          | 0.3728   |
+| 3.2121        | 23.6083 | 81000 | 3.5387          | 0.3733   |
+| 3.2298        | 23.8998 | 82000 | 3.5360          | 0.3739   |
+| 3.1542        | 24.1912 | 83000 | 3.5532          | 0.3730   |
+| 3.1837        | 24.4827 | 84000 | 3.5471          | 0.3735   |
+| 3.212         | 24.7742 | 85000 | 3.5388          | 0.3738   |
+| 3.1334        | 25.0656 | 86000 | 3.5558          | 0.3730   |
+| 3.168         | 25.3571 | 87000 | 3.5501          | 0.3733   |
+| 3.1905        | 25.6486 | 88000 | 3.5455          | 0.3734   |
+| 3.2034        | 25.9401 | 89000 | 3.5347          | 0.3745   |
+| 3.1402        | 26.2314 | 90000 | 3.5523          | 0.3736   |
+| 3.1817        | 26.5229 | 91000 | 3.5460          | 0.3740   |
+| 3.2001        | 26.8144 | 92000 | 3.5371          | 0.3744   |
+| 3.1347        | 27.1058 | 93000 | 3.5538          | 0.3734   |
+| 3.1525        | 27.3973 | 94000 | 3.5473          | 0.3737   |
+| 3.1895        | 27.6888 | 95000 | 3.5404          | 0.3740   |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_last_to_drop_3591/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 27.68880079286422,
+    "eval_accuracy": 0.3692401614472582,
+    "eval_loss": 3.5610601902008057,
+    "eval_runtime": 84.0101,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 197.917,
+    "eval_steps_per_second": 12.379,
+    "perplexity": 35.20049659090859,
+    "total_flos": 1.985593686884352e+18,
+    "train_loss": 3.4063246938605056,
+    "train_runtime": 72914.9445,
+    "train_samples": 274448,
+    "train_samples_per_second": 188.197,
+    "train_steps_per_second": 2.353
+}

exp2b/swap_0.3_last_to_drop_3591/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_last_to_drop_3591/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 27.68880079286422,
+    "eval_accuracy": 0.3692401614472582,
+    "eval_loss": 3.5610601902008057,
+    "eval_runtime": 84.0101,
+    "eval_samples": 16627,
+    "eval_samples_per_second": 197.917,
+    "eval_steps_per_second": 12.379,
+    "perplexity": 35.20049659090859
+}

exp2b/swap_0.3_last_to_drop_3591/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_last_to_drop_3591/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_drop_3591/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_last_to_drop_3591/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 27.68880079286422,
+    "total_flos": 1.985593686884352e+18,
+    "train_loss": 3.4063246938605056,
+    "train_runtime": 72914.9445,
+    "train_samples": 274448,
+    "train_samples_per_second": 188.197,
+    "train_steps_per_second": 2.353
+}