Add files using upload-large-folder tool

exp2b/swap_0.3_resemble_to_carry_1032/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_1032/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_1032/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 26.814434792747626,
+    "total_flos": 1.922817231028224e+18,
+    "train_loss": 3.4245453806338104,
+    "train_runtime": 41033.5577,
+    "train_samples": 274437,
+    "train_samples_per_second": 334.406,
+    "train_steps_per_second": 4.181
+}

exp2b/swap_0.3_resemble_to_carry_2128/README.md

@@ -0,0 +1,163 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_resemble_to_carry_2128
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/ihqh4rv7)
+# exceptions_exp2_swap_0.3_resemble_to_carry_2128
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5693
+- Accuracy: 0.3729
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 2128
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Accuracy | Validation Loss |
+|:-------------:|:-------:|:------:|:--------:|:---------------:|
+| 4.8254        | 0.2915  | 1000   | 0.2520   | 4.7814          |
+| 4.3429        | 0.5830  | 2000   | 0.2989   | 4.2852          |
+| 4.1495        | 0.8745  | 3000   | 0.3148   | 4.1004          |
+| 3.9961        | 1.1659  | 4000   | 0.3242   | 3.9964          |
+| 3.9501        | 1.4574  | 5000   | 0.3307   | 3.9212          |
+| 3.8819        | 1.7488  | 6000   | 0.3358   | 3.8631          |
+| 3.746         | 2.0402  | 7000   | 0.3404   | 3.8231          |
+| 3.761         | 2.3317  | 8000   | 0.3434   | 3.7897          |
+| 3.7421        | 2.6232  | 9000   | 0.3460   | 3.7618          |
+| 3.7268        | 2.9147  | 10000  | 0.3484   | 3.7356          |
+| 3.6436        | 3.2061  | 11000  | 0.3502   | 3.7209          |
+| 3.6544        | 3.4976  | 12000  | 0.3521   | 3.7036          |
+| 3.6477        | 3.7891  | 13000  | 0.3538   | 3.6851          |
+| 3.5517        | 4.0805  | 14000  | 0.3549   | 3.6776          |
+| 3.5819        | 4.3719  | 15000  | 0.3560   | 3.6682          |
+| 3.5693        | 4.6634  | 16000  | 0.3573   | 3.6542          |
+| 3.5879        | 4.9549  | 17000  | 0.3585   | 3.6396          |
+| 3.4941        | 5.2463  | 18000  | 0.3587   | 3.6452          |
+| 3.5351        | 5.5378  | 19000  | 0.3600   | 3.6341          |
+| 3.5235        | 5.8293  | 20000  | 0.3613   | 3.6208          |
+| 3.4477        | 6.1207  | 21000  | 0.3613   | 3.6251          |
+| 3.4856        | 6.4122  | 22000  | 0.3621   | 3.6170          |
+| 3.4815        | 6.7037  | 23000  | 0.3628   | 3.6062          |
+| 3.5027        | 6.9952  | 24000  | 0.3635   | 3.5982          |
+| 3.4276        | 7.2865  | 25000  | 0.3637   | 3.6062          |
+| 3.4657        | 7.5780  | 26000  | 0.3640   | 3.5983          |
+| 3.4597        | 7.8695  | 27000  | 0.3648   | 3.5885          |
+| 3.3868        | 8.1609  | 28000  | 0.3647   | 3.5994          |
+| 3.4322        | 8.4524  | 29000  | 0.3651   | 3.5921          |
+| 3.4325        | 8.7439  | 30000  | 0.3658   | 3.5838          |
+| 3.3427        | 9.0353  | 31000  | 0.3658   | 3.5906          |
+| 3.3823        | 9.3268  | 32000  | 0.3661   | 3.5883          |
+| 3.4155        | 9.6183  | 33000  | 0.3667   | 3.5786          |
+| 3.4158        | 9.9098  | 34000  | 0.3674   | 3.5709          |
+| 3.3389        | 10.2011 | 35000  | 0.3672   | 3.5799          |
+| 3.3614        | 10.4926 | 36000  | 0.3672   | 3.5778          |
+| 3.3803        | 10.7841 | 37000  | 0.3679   | 3.5677          |
+| 3.289         | 11.0755 | 38000  | 0.3677   | 3.5783          |
+| 3.3381        | 11.3670 | 39000  | 0.3676   | 3.5754          |
+| 3.3723        | 11.6585 | 40000  | 0.3686   | 3.5656          |
+| 3.374         | 11.9500 | 41000  | 0.3692   | 3.5579          |
+| 3.311         | 12.2414 | 42000  | 0.3684   | 3.5747          |
+| 3.35          | 12.5329 | 43000  | 0.3690   | 3.5670          |
+| 3.3499        | 12.8243 | 44000  | 0.3697   | 3.5552          |
+| 3.2678        | 13.1157 | 45000  | 0.3690   | 3.5724          |
+| 3.3057        | 13.4072 | 46000  | 0.3692   | 3.5654          |
+| 3.3354        | 13.6987 | 47000  | 0.3696   | 3.5596          |
+| 3.3392        | 13.9902 | 48000  | 0.3703   | 3.5502          |
+| 3.2776        | 14.2816 | 49000  | 0.3700   | 3.5625          |
+| 3.3019        | 14.5731 | 50000  | 0.3702   | 3.5588          |
+| 3.337         | 14.8646 | 51000  | 0.3706   | 3.5505          |
+| 3.2535        | 15.1559 | 52000  | 0.3698   | 3.5664          |
+| 3.2812        | 15.4474 | 53000  | 0.3705   | 3.5595          |
+| 3.31          | 15.7389 | 54000  | 0.3708   | 3.5518          |
+| 3.2068        | 16.0303 | 55000  | 0.3704   | 3.5636          |
+| 3.2614        | 16.3218 | 56000  | 0.3705   | 3.5616          |
+| 3.2864        | 16.6133 | 57000  | 0.3709   | 3.5545          |
+| 3.3048        | 16.9048 | 58000  | 0.3716   | 3.5447          |
+| 3.2405        | 17.1962 | 59000  | 0.3709   | 3.5626          |
+| 3.2623        | 17.4877 | 60000  | 0.3711   | 3.5550          |
+| 3.2825        | 17.7792 | 61000  | 0.3715   | 3.5491          |
+| 3.2008        | 18.0705 | 62000  | 0.3714   | 3.5612          |
+| 3.2395        | 18.3620 | 63000  | 0.3712   | 3.5590          |
+| 3.2585        | 18.6535 | 64000  | 0.3716   | 3.5504          |
+| 3.2839        | 18.9450 | 65000  | 0.3721   | 3.5440          |
+| 3.2091        | 19.2364 | 66000  | 0.3713   | 3.5641          |
+| 3.2486        | 19.5279 | 67000  | 0.3719   | 3.5526          |
+| 3.2699        | 19.8194 | 68000  | 0.3724   | 3.5451          |
+| 3.1871        | 20.1108 | 69000  | 0.3714   | 3.5659          |
+| 3.2314        | 20.4023 | 70000  | 0.3718   | 3.5571          |
+| 3.2536        | 20.6938 | 71000  | 0.3722   | 3.5493          |
+| 3.2651        | 20.9853 | 72000  | 0.3728   | 3.5397          |
+| 3.2028        | 21.2766 | 73000  | 0.3716   | 3.5608          |
+| 3.228         | 21.5681 | 74000  | 0.3725   | 3.5489          |
+| 3.2562        | 21.8596 | 75000  | 0.3726   | 3.5433          |
+| 3.1677        | 22.1510 | 76000  | 0.3720   | 3.5633          |
+| 3.2113        | 22.4425 | 77000  | 0.3722   | 3.5576          |
+| 3.2368        | 22.7340 | 78000  | 0.3726   | 3.5471          |
+| 3.1288        | 23.0254 | 79000  | 0.3721   | 3.5624          |
+| 3.1875        | 23.3169 | 80000  | 0.3719   | 3.5603          |
+| 3.1866        | 23.6083 | 81000  | 3.5600   | 0.3719          |
+| 3.2025        | 23.8998 | 82000  | 3.5537   | 0.3727          |
+| 3.1717        | 24.1915 | 83000  | 3.5624   | 0.3726          |
+| 3.1968        | 24.4830 | 84000  | 3.5568   | 0.3726          |
+| 3.2125        | 24.7745 | 85000  | 3.5489   | 0.3729          |
+| 3.1323        | 25.0659 | 86000  | 3.5694   | 0.3721          |
+| 3.1738        | 25.3574 | 87000  | 3.5588   | 0.3728          |
+| 3.1948        | 25.6489 | 88000  | 3.5495   | 0.3730          |
+| 3.2198        | 25.9404 | 89000  | 3.5441   | 0.3737          |
+| 3.1477        | 26.2317 | 90000  | 3.5598   | 0.3727          |
+| 3.188         | 26.5232 | 91000  | 3.5524   | 0.3730          |
+| 3.1925        | 26.8147 | 92000  | 3.5479   | 0.3734          |
+| 3.1235        | 27.1061 | 93000  | 3.5655   | 0.3726          |
+| 3.1649        | 27.3976 | 94000  | 3.5588   | 0.3728          |
+| 3.1772        | 27.6891 | 95000  | 3.5503   | 0.3733          |
+| 3.2038        | 27.9806 | 96000  | 3.5452   | 0.3737          |
+| 3.1323        | 28.2720 | 97000  | 3.5606   | 0.3729          |
+| 3.1696        | 28.5635 | 98000  | 3.5562   | 0.3732          |
+| 3.1698        | 28.8550 | 99000  | 3.5474   | 0.3738          |
+| 3.1194        | 29.1463 | 100000 | 3.5693   | 0.3729          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_resemble_to_carry_2128/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 29.14633008803125,
+    "eval_accuracy": 0.372863061727651,
+    "eval_loss": 3.5693490505218506,
+    "eval_runtime": 84.4707,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 196.802,
+    "eval_steps_per_second": 12.3,
+    "perplexity": 35.49348116703405,
+    "total_flos": 2.090031328198656e+18,
+    "train_loss": 0.6345674703979493,
+    "train_runtime": 15065.6637,
+    "train_samples": 274437,
+    "train_samples_per_second": 910.803,
+    "train_steps_per_second": 11.387
+}

exp2b/swap_0.3_resemble_to_carry_2128/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_resemble_to_carry_2128/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 29.14633008803125,
+    "eval_accuracy": 0.372863061727651,
+    "eval_loss": 3.5693490505218506,
+    "eval_runtime": 84.4707,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 196.802,
+    "eval_steps_per_second": 12.3,
+    "perplexity": 35.49348116703405
+}

exp2b/swap_0.3_resemble_to_carry_2128/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_resemble_to_carry_2128/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_2128/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_2128/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 29.14633008803125,
+    "total_flos": 2.090031328198656e+18,
+    "train_loss": 0.6345674703979493,
+    "train_runtime": 15065.6637,
+    "train_samples": 274437,
+    "train_samples_per_second": 910.803,
+    "train_steps_per_second": 11.387
+}

exp2b/swap_0.3_resemble_to_carry_3591/README.md

@@ -0,0 +1,155 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_resemble_to_carry_3591
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/4qo0bb73)
+# exceptions_exp2_swap_0.3_resemble_to_carry_3591
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5627
+- Accuracy: 0.3686
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 3591
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Validation Loss | Accuracy |
+|:-------------:|:-------:|:-----:|:---------------:|:--------:|
+| 4.8611        | 0.2915  | 1000  | 4.7759          | 0.2515   |
+| 4.3524        | 0.5830  | 2000  | 4.2887          | 0.2982   |
+| 4.1388        | 0.8745  | 3000  | 4.0997          | 0.3149   |
+| 3.9954        | 1.1659  | 4000  | 3.9942          | 0.3244   |
+| 3.9381        | 1.4574  | 5000  | 3.9175          | 0.3313   |
+| 3.8796        | 1.7488  | 6000  | 3.8619          | 0.3364   |
+| 3.74          | 2.0402  | 7000  | 3.8180          | 0.3409   |
+| 3.7535        | 2.3317  | 8000  | 3.7894          | 0.3437   |
+| 3.7476        | 2.6232  | 9000  | 3.7562          | 0.3464   |
+| 3.7247        | 2.9147  | 10000 | 3.7313          | 0.3487   |
+| 3.6212        | 3.2061  | 11000 | 3.7204          | 0.3508   |
+| 3.6546        | 3.4976  | 12000 | 3.7027          | 0.3522   |
+| 3.6366        | 3.7891  | 13000 | 3.6812          | 0.3541   |
+| 3.539         | 4.0805  | 14000 | 3.6769          | 0.3553   |
+| 3.5748        | 4.3719  | 15000 | 3.6634          | 0.3563   |
+| 3.5716        | 4.6634  | 16000 | 3.6501          | 0.3576   |
+| 3.5806        | 4.9549  | 17000 | 3.6361          | 0.3589   |
+| 3.5099        | 5.2463  | 18000 | 3.6429          | 0.3595   |
+| 3.535         | 5.5378  | 19000 | 3.6311          | 0.3601   |
+| 3.512         | 5.8293  | 20000 | 3.6188          | 0.3613   |
+| 3.4448        | 6.1207  | 21000 | 3.6224          | 0.3615   |
+| 3.475         | 6.4122  | 22000 | 3.6144          | 0.3623   |
+| 3.5005        | 6.7037  | 23000 | 3.6068          | 0.3628   |
+| 3.5028        | 6.9952  | 24000 | 3.5979          | 0.3639   |
+| 3.443         | 7.2865  | 25000 | 3.6031          | 0.3638   |
+| 3.453         | 7.5780  | 26000 | 3.5940          | 0.3647   |
+| 3.4619        | 7.8695  | 27000 | 3.5868          | 0.3650   |
+| 3.4049        | 8.1609  | 28000 | 3.5938          | 0.3652   |
+| 3.4264        | 8.4524  | 29000 | 3.5892          | 0.3652   |
+| 3.4498        | 8.7439  | 30000 | 3.5790          | 0.3660   |
+| 3.3344        | 9.0353  | 31000 | 3.5842          | 0.3663   |
+| 3.3909        | 9.3268  | 32000 | 3.5841          | 0.3664   |
+| 3.4037        | 9.6183  | 33000 | 3.5758          | 0.3669   |
+| 3.4071        | 9.9098  | 34000 | 3.5660          | 0.3677   |
+| 3.3266        | 10.2011 | 35000 | 3.5796          | 0.3672   |
+| 3.3795        | 10.4926 | 36000 | 3.5754          | 0.3676   |
+| 3.3768        | 10.7841 | 37000 | 3.5650          | 0.3682   |
+| 3.3094        | 11.0755 | 38000 | 3.5742          | 0.3679   |
+| 3.3421        | 11.3670 | 39000 | 3.5725          | 0.3681   |
+| 3.3665        | 11.6585 | 40000 | 3.5627          | 0.3686   |
+| 3.379         | 11.9500 | 41000 | 3.5567          | 0.3692   |
+| 3.2988        | 12.2414 | 42000 | 3.5693          | 0.3690   |
+| 3.346         | 12.5329 | 43000 | 3.5595          | 0.3694   |
+| 3.3564        | 12.8243 | 44000 | 3.5514          | 0.3701   |
+| 3.2784        | 13.1157 | 45000 | 3.5658          | 0.3694   |
+| 3.3288        | 13.4072 | 46000 | 3.5639          | 0.3696   |
+| 3.3299        | 13.6987 | 47000 | 3.5541          | 0.3700   |
+| 3.3462        | 13.9902 | 48000 | 3.5466          | 0.3708   |
+| 3.2894        | 14.2816 | 49000 | 3.5607          | 0.3700   |
+| 3.3081        | 14.5731 | 50000 | 3.5544          | 0.3706   |
+| 3.327         | 14.8646 | 51000 | 3.5506          | 0.3709   |
+| 3.2482        | 15.1559 | 52000 | 3.5612          | 0.3704   |
+| 3.2802        | 15.4474 | 53000 | 3.5576          | 0.3704   |
+| 3.3049        | 15.7389 | 54000 | 3.5509          | 0.3709   |
+| 3.2122        | 16.0303 | 55000 | 3.5596          | 0.3706   |
+| 3.2658        | 16.3218 | 56000 | 3.5581          | 0.3710   |
+| 3.2865        | 16.6133 | 57000 | 3.5490          | 0.3713   |
+| 3.2986        | 16.9048 | 58000 | 3.5423          | 0.3717   |
+| 3.2354        | 17.1962 | 59000 | 3.5566          | 0.3712   |
+| 3.2642        | 17.4877 | 60000 | 3.5545          | 0.3714   |
+| 3.2777        | 17.7792 | 61000 | 3.5483          | 0.3716   |
+| 3.2038        | 18.0705 | 62000 | 3.5558          | 0.3715   |
+| 3.2268        | 18.3620 | 63000 | 3.5541          | 0.3715   |
+| 3.2634        | 18.6535 | 64000 | 3.5468          | 0.3721   |
+| 3.2768        | 18.9450 | 65000 | 3.5386          | 0.3724   |
+| 3.2221        | 19.2364 | 66000 | 3.5558          | 0.3717   |
+| 3.243         | 19.5279 | 67000 | 3.5519          | 0.3722   |
+| 3.2603        | 19.8194 | 68000 | 3.5451          | 0.3723   |
+| 3.1875        | 20.1108 | 69000 | 3.5612          | 0.3717   |
+| 3.2295        | 20.4023 | 70000 | 3.5522          | 0.3721   |
+| 3.25          | 20.6938 | 71000 | 3.5442          | 0.3725   |
+| 3.2594        | 20.9853 | 72000 | 3.5358          | 0.3734   |
+| 3.2081        | 21.2766 | 73000 | 3.5549          | 0.3719   |
+| 3.2195        | 21.5681 | 74000 | 3.5473          | 0.3726   |
+| 3.2397        | 21.8596 | 75000 | 3.5381          | 0.3732   |
+| 3.1756        | 22.1510 | 76000 | 3.5561          | 0.3721   |
+| 3.2106        | 22.4425 | 77000 | 3.5478          | 0.3727   |
+| 3.2176        | 22.7340 | 78000 | 3.5441          | 0.3728   |
+| 3.1459        | 23.0254 | 79000 | 3.5531          | 0.3725   |
+| 3.1883        | 23.3169 | 80000 | 3.5532          | 0.3726   |
+| 3.2169        | 23.6083 | 81000 | 3.5488          | 0.3728   |
+| 3.2302        | 23.8998 | 82000 | 3.5391          | 0.3733   |
+| 3.1652        | 24.1912 | 83000 | 3.5565          | 0.3726   |
+| 3.1966        | 24.4827 | 84000 | 3.5498          | 0.3731   |
+| 3.2087        | 24.7742 | 85000 | 3.5395          | 0.3737   |
+| 3.1184        | 25.0656 | 86000 | 3.5565          | 0.3727   |
+| 3.1774        | 25.3571 | 87000 | 3.5512          | 0.3732   |
+| 3.1971        | 25.6486 | 88000 | 3.5430          | 0.3735   |
+| 3.2131        | 25.9401 | 89000 | 3.5372          | 0.3739   |
+| 3.15          | 26.2314 | 90000 | 3.5535          | 0.3731   |
+| 3.1785        | 26.5229 | 91000 | 3.5473          | 0.3733   |
+| 3.183         | 26.8144 | 92000 | 3.5399          | 0.3740   |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_resemble_to_carry_3591/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 26.814434792747626,
+    "eval_accuracy": 0.36857199832745996,
+    "eval_loss": 3.5626509189605713,
+    "eval_runtime": 53.3428,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 311.645,
+    "eval_steps_per_second": 19.478,
+    "perplexity": 35.25653559280293,
+    "total_flos": 1.922817231028224e+18,
+    "train_loss": 3.415760473334271,
+    "train_runtime": 40855.253,
+    "train_samples": 274437,
+    "train_samples_per_second": 335.865,
+    "train_steps_per_second": 4.199
+}

exp2b/swap_0.3_resemble_to_carry_3591/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_resemble_to_carry_3591/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 26.814434792747626,
+    "eval_accuracy": 0.36857199832745996,
+    "eval_loss": 3.5626509189605713,
+    "eval_runtime": 53.3428,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 311.645,
+    "eval_steps_per_second": 19.478,
+    "perplexity": 35.25653559280293
+}

exp2b/swap_0.3_resemble_to_carry_3591/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_resemble_to_carry_3591/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_3591/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_3591/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 26.814434792747626,
+    "total_flos": 1.922817231028224e+18,
+    "train_loss": 3.415760473334271,
+    "train_runtime": 40855.253,
+    "train_samples": 274437,
+    "train_samples_per_second": 335.865,
+    "train_steps_per_second": 4.199
+}

exp2b/swap_0.3_resemble_to_carry_40817/README.md

@@ -0,0 +1,155 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_resemble_to_carry_40817
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/8ylpi7e4)
+# exceptions_exp2_swap_0.3_resemble_to_carry_40817
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5596
+- Accuracy: 0.3690
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 40817
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Validation Loss | Accuracy |
+|:-------------:|:-------:|:-----:|:---------------:|:--------:|
+| 4.8345        | 0.2915  | 1000  | 4.7542          | 0.2547   |
+| 4.3449        | 0.5830  | 2000  | 4.2843          | 0.2988   |
+| 4.1462        | 0.8745  | 3000  | 4.1035          | 0.3147   |
+| 4.0076        | 1.1659  | 4000  | 3.9945          | 0.3245   |
+| 3.9377        | 1.4574  | 5000  | 3.9197          | 0.3310   |
+| 3.8703        | 1.7488  | 6000  | 3.8609          | 0.3368   |
+| 3.7494        | 2.0402  | 7000  | 3.8190          | 0.3407   |
+| 3.7645        | 2.3317  | 8000  | 3.7898          | 0.3434   |
+| 3.7388        | 2.6232  | 9000  | 3.7587          | 0.3464   |
+| 3.7327        | 2.9147  | 10000 | 3.7339          | 0.3488   |
+| 3.6382        | 3.2061  | 11000 | 3.7175          | 0.3507   |
+| 3.6557        | 3.4976  | 12000 | 3.7020          | 0.3524   |
+| 3.6498        | 3.7891  | 13000 | 3.6843          | 0.3540   |
+| 3.5505        | 4.0805  | 14000 | 3.6781          | 0.3555   |
+| 3.569         | 4.3719  | 15000 | 3.6661          | 0.3562   |
+| 3.5854        | 4.6634  | 16000 | 3.6523          | 0.3577   |
+| 3.5923        | 4.9549  | 17000 | 3.6382          | 0.3588   |
+| 3.5001        | 5.2463  | 18000 | 3.6402          | 0.3592   |
+| 3.517         | 5.5378  | 19000 | 3.6296          | 0.3603   |
+| 3.5252        | 5.8293  | 20000 | 3.6210          | 0.3612   |
+| 3.4564        | 6.1207  | 21000 | 3.6203          | 0.3617   |
+| 3.4801        | 6.4122  | 22000 | 3.6164          | 0.3618   |
+| 3.4882        | 6.7037  | 23000 | 3.6030          | 0.3630   |
+| 3.4884        | 6.9952  | 24000 | 3.5947          | 0.3636   |
+| 3.4314        | 7.2865  | 25000 | 3.6041          | 0.3637   |
+| 3.4486        | 7.5780  | 26000 | 3.5945          | 0.3643   |
+| 3.4581        | 7.8695  | 27000 | 3.5867          | 0.3653   |
+| 3.3706        | 8.1609  | 28000 | 3.5964          | 0.3650   |
+| 3.4142        | 8.4524  | 29000 | 3.5873          | 0.3656   |
+| 3.428         | 8.7439  | 30000 | 3.5790          | 0.3663   |
+| 3.3208        | 9.0353  | 31000 | 3.5875          | 0.3658   |
+| 3.3859        | 9.3268  | 32000 | 3.5835          | 0.3666   |
+| 3.3965        | 9.6183  | 33000 | 3.5745          | 0.3672   |
+| 3.4162        | 9.9098  | 34000 | 3.5682          | 0.3679   |
+| 3.3426        | 10.2011 | 35000 | 3.5803          | 0.3671   |
+| 3.3556        | 10.4926 | 36000 | 3.5737          | 0.3677   |
+| 3.3876        | 10.7841 | 37000 | 3.5629          | 0.3683   |
+| 3.286         | 11.0755 | 38000 | 3.5747          | 0.3684   |
+| 3.3278        | 11.3670 | 39000 | 3.5720          | 0.3683   |
+| 3.3631        | 11.6585 | 40000 | 3.5596          | 0.3690   |
+| 3.3638        | 11.9500 | 41000 | 3.5536          | 0.3695   |
+| 3.3048        | 12.2414 | 42000 | 3.5683          | 0.3689   |
+| 3.335         | 12.5329 | 43000 | 3.5603          | 0.3696   |
+| 3.3367        | 12.8243 | 44000 | 3.5529          | 0.3699   |
+| 3.2809        | 13.1157 | 45000 | 3.5675          | 0.3694   |
+| 3.3053        | 13.4072 | 46000 | 3.5603          | 0.3697   |
+| 3.3302        | 13.6987 | 47000 | 3.5534          | 0.3700   |
+| 3.344         | 13.9902 | 48000 | 3.5466          | 0.3707   |
+| 3.277         | 14.2816 | 49000 | 3.5646          | 0.3699   |
+| 3.2929        | 14.5731 | 50000 | 3.5584          | 0.3705   |
+| 3.3322        | 14.8646 | 51000 | 3.5469          | 0.3712   |
+| 3.2444        | 15.1559 | 52000 | 3.5634          | 0.3704   |
+| 3.2805        | 15.4474 | 53000 | 3.5555          | 0.3706   |
+| 3.305         | 15.7389 | 54000 | 3.5488          | 0.3712   |
+| 3.2108        | 16.0303 | 55000 | 3.5609          | 0.3709   |
+| 3.2638        | 16.3218 | 56000 | 3.5584          | 0.3711   |
+| 3.2741        | 16.6133 | 57000 | 3.5506          | 0.3715   |
+| 3.3075        | 16.9048 | 58000 | 3.5400          | 0.3718   |
+| 3.233         | 17.1962 | 59000 | 3.5565          | 0.3712   |
+| 3.2453        | 17.4877 | 60000 | 3.5496          | 0.3716   |
+| 3.2754        | 17.7792 | 61000 | 3.5436          | 0.3723   |
+| 3.1915        | 18.0705 | 62000 | 3.5596          | 0.3717   |
+| 3.2425        | 18.3620 | 63000 | 3.5529          | 0.3719   |
+| 3.2634        | 18.6535 | 64000 | 3.5452          | 0.3722   |
+| 3.2801        | 18.9450 | 65000 | 3.5382          | 0.3725   |
+| 3.2169        | 19.2364 | 66000 | 3.5543          | 0.3715   |
+| 3.2421        | 19.5279 | 67000 | 3.5504          | 0.3719   |
+| 3.2546        | 19.8194 | 68000 | 3.5404          | 0.3727   |
+| 3.1893        | 20.1108 | 69000 | 3.5546          | 0.3719   |
+| 3.236         | 20.4023 | 70000 | 3.5506          | 0.3724   |
+| 3.2453        | 20.6938 | 71000 | 3.5430          | 0.3727   |
+| 3.2504        | 20.9853 | 72000 | 3.5360          | 0.3734   |
+| 3.2132        | 21.2766 | 73000 | 3.5544          | 0.3723   |
+| 3.2249        | 21.5681 | 74000 | 3.5482          | 0.3725   |
+| 3.2584        | 21.8596 | 75000 | 3.5401          | 0.3733   |
+| 3.1766        | 22.1510 | 76000 | 3.5549          | 0.3725   |
+| 3.1987        | 22.4425 | 77000 | 3.5494          | 0.3730   |
+| 3.223         | 22.7340 | 78000 | 3.5433          | 0.3732   |
+| 3.1401        | 23.0254 | 79000 | 3.5546          | 0.3726   |
+| 3.1789        | 23.3169 | 80000 | 3.5521          | 0.3728   |
+| 3.2151        | 23.6083 | 81000 | 3.5446          | 0.3732   |
+| 3.2234        | 23.8998 | 82000 | 3.5374          | 0.3738   |
+| 3.1593        | 24.1912 | 83000 | 3.5592          | 0.3728   |
+| 3.1859        | 24.4827 | 84000 | 3.5463          | 0.3734   |
+| 3.2163        | 24.7742 | 85000 | 3.5389          | 0.3739   |
+| 3.1289        | 25.0656 | 86000 | 3.5566          | 0.3730   |
+| 3.1688        | 25.3571 | 87000 | 3.5506          | 0.3733   |
+| 3.1849        | 25.6486 | 88000 | 3.5469          | 0.3737   |
+| 3.2094        | 25.9401 | 89000 | 3.5399          | 0.3738   |
+| 3.1548        | 26.2314 | 90000 | 3.5577          | 0.3731   |
+| 3.1747        | 26.5229 | 91000 | 3.5498          | 0.3736   |
+| 3.1912        | 26.8144 | 92000 | 3.5404          | 0.3742   |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_resemble_to_carry_40817/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 26.814434792747626,
+    "eval_accuracy": 0.36904922786285926,
+    "eval_loss": 3.559636354446411,
+    "eval_runtime": 54.5086,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 304.979,
+    "eval_steps_per_second": 19.061,
+    "perplexity": 35.15041252949002,
+    "total_flos": 1.922817231028224e+18,
+    "train_loss": 3.4145377583711043,
+    "train_runtime": 40944.2836,
+    "train_samples": 274437,
+    "train_samples_per_second": 335.135,
+    "train_steps_per_second": 4.19
+}

exp2b/swap_0.3_resemble_to_carry_40817/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_resemble_to_carry_40817/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 26.814434792747626,
+    "eval_accuracy": 0.36904922786285926,
+    "eval_loss": 3.559636354446411,
+    "eval_runtime": 54.5086,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 304.979,
+    "eval_steps_per_second": 19.061,
+    "perplexity": 35.15041252949002
+}

exp2b/swap_0.3_resemble_to_carry_40817/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_resemble_to_carry_40817/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_40817/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp2b/swap_0.3_resemble_to_carry_40817/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 26.814434792747626,
+    "total_flos": 1.922817231028224e+18,
+    "train_loss": 3.4145377583711043,
+    "train_runtime": 40944.2836,
+    "train_samples": 274437,
+    "train_samples_per_second": 335.135,
+    "train_steps_per_second": 4.19
+}

exp2b/swap_0.3_resemble_to_carry_5039/README.md

@@ -0,0 +1,172 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp2_swap_0.3_resemble_to_carry_5039
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp2/runs/tx6feeku)
+# exceptions_exp2_swap_0.3_resemble_to_carry_5039
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5631
+- Accuracy: 0.3690
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 5039
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step   | Validation Loss | Accuracy |
+|:-------------:|:-------:|:------:|:---------------:|:--------:|
+| 4.8406        | 0.2915  | 1000   | 4.7602          | 0.2542   |
+| 4.3426        | 0.5830  | 2000   | 4.2857          | 0.2993   |
+| 4.1584        | 0.8745  | 3000   | 4.1017          | 0.3153   |
+| 3.9933        | 1.1659  | 4000   | 3.9941          | 0.3247   |
+| 3.9371        | 1.4574  | 5000   | 3.9185          | 0.3314   |
+| 3.8857        | 1.7488  | 6000   | 3.8605          | 0.3367   |
+| 3.7556        | 2.0402  | 7000   | 3.8168          | 0.3410   |
+| 3.7608        | 2.3317  | 8000   | 3.7872          | 0.3439   |
+| 3.7568        | 2.6232  | 9000   | 3.7561          | 0.3467   |
+| 3.7268        | 2.9147  | 10000  | 3.7327          | 0.3491   |
+| 3.6451        | 3.2061  | 11000  | 3.7218          | 0.3510   |
+| 3.6454        | 3.4976  | 12000  | 3.7010          | 0.3525   |
+| 3.6472        | 3.7891  | 13000  | 3.6830          | 0.3542   |
+| 3.5317        | 4.0805  | 14000  | 3.6755          | 0.3555   |
+| 3.5809        | 4.3719  | 15000  | 3.6661          | 0.3565   |
+| 3.5804        | 4.6634  | 16000  | 3.6525          | 0.3578   |
+| 3.5827        | 4.9549  | 17000  | 3.6376          | 0.3589   |
+| 3.5057        | 5.2463  | 18000  | 3.6420          | 0.3593   |
+| 3.5256        | 5.5378  | 19000  | 3.6301          | 0.3602   |
+| 3.5254        | 5.8293  | 20000  | 3.6214          | 0.3613   |
+| 3.456         | 6.1207  | 21000  | 3.6228          | 0.3616   |
+| 3.4779        | 6.4122  | 22000  | 3.6160          | 0.3620   |
+| 3.4894        | 6.7037  | 23000  | 3.6025          | 0.3634   |
+| 3.4984        | 6.9952  | 24000  | 3.5956          | 0.3643   |
+| 3.432         | 7.2865  | 25000  | 3.6050          | 0.3640   |
+| 3.4586        | 7.5780  | 26000  | 3.5945          | 0.3644   |
+| 3.4597        | 7.8695  | 27000  | 3.5886          | 0.3654   |
+| 3.3844        | 8.1609  | 28000  | 3.5971          | 0.3651   |
+| 3.4176        | 8.4524  | 29000  | 3.5893          | 0.3655   |
+| 3.4384        | 8.7439  | 30000  | 3.5832          | 0.3661   |
+| 3.3243        | 9.0353  | 31000  | 3.5843          | 0.3663   |
+| 3.3782        | 9.3268  | 32000  | 3.5828          | 0.3665   |
+| 3.3918        | 9.6183  | 33000  | 3.5766          | 0.3672   |
+| 3.4126        | 9.9098  | 34000  | 3.5674          | 0.3678   |
+| 3.3394        | 10.2011 | 35000  | 3.5817          | 0.3675   |
+| 3.386         | 10.4926 | 36000  | 3.5728          | 0.3679   |
+| 3.3937        | 10.7841 | 37000  | 3.5647          | 0.3683   |
+| 3.3096        | 11.0755 | 38000  | 3.5749          | 0.3680   |
+| 3.3584        | 11.3670 | 39000  | 3.5718          | 0.3683   |
+| 3.3798        | 11.6585 | 40000  | 3.5631          | 0.3690   |
+| 3.3761        | 11.9500 | 41000  | 3.5563          | 0.3694   |
+| 3.3134        | 12.2414 | 42000  | 3.5714          | 0.3690   |
+| 3.3291        | 12.5329 | 43000  | 3.5649          | 0.3693   |
+| 3.3643        | 12.8243 | 44000  | 3.5551          | 0.3699   |
+| 3.2777        | 13.1157 | 45000  | 3.5666          | 0.3694   |
+| 3.324         | 13.4072 | 46000  | 3.5604          | 0.3697   |
+| 3.3412        | 13.6987 | 47000  | 3.5577          | 0.3701   |
+| 3.3436        | 13.9902 | 48000  | 3.5480          | 0.3706   |
+| 3.2886        | 14.2816 | 49000  | 3.5631          | 0.3700   |
+| 3.3235        | 14.5731 | 50000  | 3.5542          | 0.3707   |
+| 3.3201        | 14.8646 | 51000  | 3.5472          | 0.3713   |
+| 3.2554        | 15.1559 | 52000  | 3.5628          | 0.3706   |
+| 3.297         | 15.4474 | 53000  | 3.5557          | 0.3708   |
+| 3.3105        | 15.7389 | 54000  | 3.5499          | 0.3714   |
+| 3.2041        | 16.0303 | 55000  | 3.5580          | 0.3710   |
+| 3.2529        | 16.3218 | 56000  | 3.5588          | 0.3712   |
+| 3.2937        | 16.6133 | 57000  | 3.5506          | 0.3715   |
+| 3.2971        | 16.9048 | 58000  | 3.5438          | 0.3722   |
+| 3.2371        | 17.1962 | 59000  | 3.5618          | 0.3712   |
+| 3.259         | 17.4877 | 60000  | 3.5539          | 0.3715   |
+| 3.2803        | 17.7792 | 61000  | 3.5465          | 0.3719   |
+| 3.2012        | 18.0705 | 62000  | 3.5580          | 0.3716   |
+| 3.2424        | 18.3620 | 63000  | 3.5586          | 0.3715   |
+| 3.2685        | 18.6535 | 64000  | 3.5456          | 0.3722   |
+| 3.2767        | 18.9450 | 65000  | 3.5391          | 0.3728   |
+| 3.2225        | 19.2364 | 66000  | 3.5525          | 0.3721   |
+| 3.2502        | 19.5279 | 67000  | 3.5481          | 0.3723   |
+| 3.2638        | 19.8194 | 68000  | 3.5450          | 0.3726   |
+| 3.1862        | 20.1108 | 69000  | 3.5574          | 0.3721   |
+| 3.2157        | 20.4023 | 70000  | 3.5522          | 0.3722   |
+| 3.2405        | 20.6938 | 71000  | 3.5429          | 0.3729   |
+| 3.2604        | 20.9853 | 72000  | 3.5372          | 0.3732   |
+| 3.2131        | 21.2766 | 73000  | 3.5569          | 0.3721   |
+| 3.233         | 21.5681 | 74000  | 3.5494          | 0.3727   |
+| 3.2436        | 21.8596 | 75000  | 3.5398          | 0.3731   |
+| 3.1763        | 22.1510 | 76000  | 3.5562          | 0.3726   |
+| 3.2085        | 22.4425 | 77000  | 3.5491          | 0.3731   |
+| 3.2296        | 22.7340 | 78000  | 3.5412          | 0.3731   |
+| 3.1511        | 23.0254 | 79000  | 3.5563          | 0.3728   |
+| 3.1788        | 23.3169 | 80000  | 3.5531          | 0.3729   |
+| 3.2182        | 23.6083 | 81000  | 3.5454          | 0.3732   |
+| 3.2287        | 23.8998 | 82000  | 3.5393          | 0.3738   |
+| 3.1728        | 24.1912 | 83000  | 3.5567          | 0.3732   |
+| 3.2058        | 24.4827 | 84000  | 3.5507          | 0.3729   |
+| 3.2082        | 24.7742 | 85000  | 3.5400          | 0.3740   |
+| 3.1336        | 25.0656 | 86000  | 3.5551          | 0.3734   |
+| 3.1797        | 25.3571 | 87000  | 3.5509          | 0.3732   |
+| 3.2031        | 25.6486 | 88000  | 3.5446          | 0.3736   |
+| 3.2068        | 25.9401 | 89000  | 3.5353          | 0.3744   |
+| 3.154         | 26.2314 | 90000  | 3.5553          | 0.3733   |
+| 3.1807        | 26.5229 | 91000  | 3.5500          | 0.3733   |
+| 3.1839        | 26.8144 | 92000  | 3.5408          | 0.3744   |
+| 3.1251        | 27.1058 | 93000  | 3.5590          | 0.3732   |
+| 3.1469        | 27.3973 | 94000  | 3.5524          | 0.3734   |
+| 3.1806        | 27.6888 | 95000  | 3.5447          | 0.3742   |
+| 3.2011        | 27.9803 | 96000  | 3.5374          | 0.3745   |
+| 3.1189        | 28.2717 | 97000  | 3.5555          | 0.3736   |
+| 3.1639        | 28.5632 | 98000  | 3.5513          | 0.3738   |
+| 3.1851        | 28.8547 | 99000  | 3.5415          | 0.3744   |
+| 3.1165        | 29.1460 | 100000 | 3.5606          | 0.3733   |
+| 3.168         | 29.4375 | 101000 | 3.5508          | 0.3738   |
+| 3.1781        | 29.7290 | 102000 | 3.5437          | 0.3743   |
+| 3.084         | 30.0204 | 103000 | 3.5531          | 0.3739   |
+| 3.1345        | 30.3119 | 104000 | 3.5561          | 0.3738   |
+| 3.147         | 30.6034 | 105000 | 3.5460          | 0.3745   |
+| 3.174         | 30.8949 | 106000 | 3.5419          | 0.3748   |
+| 3.1096        | 31.1863 | 107000 | 3.5603          | 0.3737   |
+| 3.125         | 31.4778 | 108000 | 3.5515          | 0.3741   |
+| 3.1522        | 31.7693 | 109000 | 3.5435          | 0.3746   |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp2b/swap_0.3_resemble_to_carry_5039/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 31.76925319186148,
+    "eval_accuracy": 0.3689947243416728,
+    "eval_loss": 3.5631160736083984,
+    "eval_runtime": 53.3948,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 311.341,
+    "eval_steps_per_second": 19.459,
+    "perplexity": 35.27293914899965,
+    "total_flos": 2.278118216761344e+18,
+    "train_loss": 3.3759910841075653,
+    "train_runtime": 48511.1396,
+    "train_samples": 274437,
+    "train_samples_per_second": 282.86,
+    "train_steps_per_second": 3.536
+}

exp2b/swap_0.3_resemble_to_carry_5039/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp2b/swap_0.3_resemble_to_carry_5039/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 31.76925319186148,
+    "eval_accuracy": 0.3689947243416728,
+    "eval_loss": 3.5631160736083984,
+    "eval_runtime": 53.3948,
+    "eval_samples": 16624,
+    "eval_samples_per_second": 311.341,
+    "eval_steps_per_second": 19.459,
+    "perplexity": 35.27293914899965
+}

exp2b/swap_0.3_resemble_to_carry_5039/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp2b/swap_0.3_resemble_to_carry_5039/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}