craa commited on 2 days ago

Commit

43fd2f4

verified ·

1 Parent(s): 442d639

Add files using upload-large-folder tool

Browse files

Files changed (50) hide show

base/100M__1208/README.md +155 -0
base/100M__1208/all_results.json +16 -0
base/100M__1208/config.json +31 -0
base/100M__1208/eval_results.json +10 -0
base/100M__1208/generation_config.json +6 -0
base/100M__1208/special_tokens_map.json +1 -0
base/100M__1208/tokenizer.json +0 -0
base/100M__1208/tokenizer_config.json +0 -0
base/100M__1208/train_results.json +9 -0
base/100M__1208/trainer_state.json +0 -0
base/100M__495/README.md +155 -0
base/100M__495/all_results.json +16 -0
base/100M__495/config.json +31 -0
base/100M__495/eval_results.json +10 -0
base/100M__495/generation_config.json +6 -0
base/100M__495/special_tokens_map.json +1 -0
base/100M__495/tokenizer.json +0 -0
base/100M__495/tokenizer_config.json +0 -0
base/100M__495/train_results.json +9 -0
base/100M__495/trainer_state.json +0 -0
base/100M__634/README.md +155 -0
base/100M__634/all_results.json +16 -0
base/100M__634/config.json +31 -0
base/100M__634/eval_results.json +10 -0
base/100M__634/generation_config.json +6 -0
base/100M__634/special_tokens_map.json +1 -0
base/100M__634/tokenizer.json +0 -0
base/100M__634/tokenizer_config.json +0 -0
base/100M__634/train_results.json +9 -0
base/100M__634/trainer_state.json +0 -0
base/100M__6910/README.md +155 -0
base/100M__6910/all_results.json +16 -0
base/100M__6910/config.json +31 -0
base/100M__6910/eval_results.json +10 -0
base/100M__6910/generation_config.json +6 -0
base/100M__6910/special_tokens_map.json +1 -0
base/100M__6910/tokenizer.json +0 -0
base/100M__6910/tokenizer_config.json +0 -0
base/100M__6910/train_results.json +9 -0
base/100M__6910/trainer_state.json +0 -0
base/100M__8397/README.md +155 -0
base/100M__8397/all_results.json +16 -0
base/100M__8397/config.json +31 -0
base/100M__8397/eval_results.json +10 -0
base/100M__8397/generation_config.json +6 -0
base/100M__8397/special_tokens_map.json +1 -0
base/100M__8397/tokenizer.json +0 -0
base/100M__8397/tokenizer_config.json +0 -0
base/100M__8397/train_results.json +9 -0
base/100M__8397/trainer_state.json +0 -0

base/100M__1208/README.md ADDED Viewed

	@@ -0,0 +1,155 @@

+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: 100M__1208
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+# 100M__1208
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.4619
+- Accuracy: 0.3767
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 32
+- eval_batch_size: 16
+- seed: 1208
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50
+- mixed_precision_training: Native AMP
+### Training results
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 5.1151        | 0.1078  | 1000  | 0.2267   | 5.0271          |
+| 4.5792        | 0.2156  | 2000  | 0.2710   | 4.5047          |
+| 4.2988        | 0.3235  | 3000  | 0.2984   | 4.2390          |
+| 4.1617        | 0.4313  | 4000  | 0.3115   | 4.0934          |
+| 4.0655        | 0.5391  | 5000  | 0.3216   | 3.9903          |
+| 3.9934        | 0.6469  | 6000  | 0.3281   | 3.9243          |
+| 3.9432        | 0.7547  | 7000  | 0.3334   | 3.8671          |
+| 3.8869        | 0.8625  | 8000  | 0.3375   | 3.8181          |
+| 3.848         | 0.9704  | 9000  | 0.3415   | 3.7792          |
+| 3.7701        | 1.0782  | 10000 | 0.3447   | 3.7489          |
+| 3.7519        | 1.1860  | 11000 | 0.3467   | 3.7236          |
+| 3.7387        | 1.2938  | 12000 | 0.3488   | 3.6996          |
+| 3.7317        | 1.4016  | 13000 | 0.3512   | 3.6775          |
+| 3.6971        | 1.5094  | 14000 | 0.3538   | 3.6579          |
+| 3.6885        | 1.6173  | 15000 | 0.3551   | 3.6401          |
+| 3.6771        | 1.7251  | 16000 | 0.3570   | 3.6195          |
+| 3.6503        | 1.8329  | 17000 | 0.3589   | 3.6034          |
+| 3.6523        | 1.9407  | 18000 | 0.3600   | 3.5907          |
+| 3.5469        | 2.0485  | 19000 | 0.3615   | 3.5790          |
+| 3.553         | 2.1563  | 20000 | 0.3630   | 3.5688          |
+| 3.5623        | 2.2642  | 21000 | 0.3637   | 3.5586          |
+| 3.5464        | 2.3720  | 22000 | 0.3649   | 3.5480          |
+| 3.543         | 2.4798  | 23000 | 0.3659   | 3.5397          |
+| 3.5598        | 2.5876  | 24000 | 0.3667   | 3.5288          |
+| 3.533         | 2.6954  | 25000 | 0.3680   | 3.5185          |
+| 3.5272        | 2.8032  | 26000 | 0.3688   | 3.5118          |
+| 3.5211        | 2.9111  | 27000 | 0.3695   | 3.5022          |
+| 3.4424        | 3.0189  | 28000 | 0.3706   | 3.4983          |
+| 3.4427        | 3.1267  | 29000 | 0.3707   | 3.4975          |
+| 3.4427        | 3.2345  | 30000 | 0.3713   | 3.4904          |
+| 3.445         | 3.3423  | 31000 | 0.3723   | 3.4827          |
+| 3.4717        | 3.4501  | 32000 | 0.3730   | 3.4765          |
+| 3.4426        | 3.5580  | 33000 | 0.3736   | 3.4684          |
+| 3.4603        | 3.6658  | 34000 | 0.3741   | 3.4616          |
+| 3.4463        | 3.7736  | 35000 | 0.3749   | 3.4561          |
+| 3.4731        | 3.8814  | 36000 | 0.3756   | 3.4498          |
+| 3.4391        | 3.9892  | 37000 | 0.3763   | 3.4429          |
+| 3.3773        | 4.0970  | 38000 | 0.3766   | 3.4481          |
+| 3.3765        | 4.2049  | 39000 | 0.3770   | 3.4400          |
+| 3.3936        | 4.3127  | 40000 | 0.3779   | 3.4354          |
+| 3.3884        | 4.4205  | 41000 | 0.3781   | 3.4319          |
+| 3.3762        | 4.5283  | 42000 | 0.3790   | 3.4272          |
+| 3.375         | 4.6361  | 43000 | 0.3788   | 3.4225          |
+| 3.3747        | 4.7439  | 44000 | 0.3800   | 3.4165          |
+| 3.3985        | 4.8518  | 45000 | 0.3801   | 3.4101          |
+| 3.3853        | 4.9596  | 46000 | 0.3807   | 3.4061          |
+| 3.2829        | 5.0674  | 47000 | 0.3811   | 3.4088          |
+| 3.3263        | 5.1752  | 48000 | 0.3811   | 3.4104          |
+| 3.3098        | 5.2830  | 49000 | 0.3818   | 3.4050          |
+| 3.348         | 5.3908  | 50000 | 0.3819   | 3.4028          |
+| 3.331         | 5.4987  | 51000 | 0.3823   | 3.3962          |
+| 3.3412        | 5.6065  | 52000 | 0.3829   | 3.3931          |
+| 3.3414        | 5.7143  | 53000 | 0.3833   | 3.3883          |
+| 3.3438        | 5.8221  | 54000 | 0.3836   | 3.3822          |
+| 3.3077        | 5.9299  | 55000 | 0.3844   | 3.3785          |
+| 3.2654        | 6.0377  | 56000 | 0.3842   | 3.3808          |
+| 3.2674        | 6.1456  | 57000 | 0.3846   | 3.3812          |
+| 3.2691        | 6.2534  | 58000 | 0.3846   | 3.3798          |
+| 3.276         | 6.3612  | 59000 | 0.3851   | 3.3751          |
+| 3.2903        | 6.4690  | 60000 | 0.3858   | 3.3693          |
+| 3.2681        | 6.5768  | 61000 | 0.3861   | 3.3662          |
+| 3.2793        | 6.6846  | 62000 | 0.3863   | 3.3626          |
+| 3.2753        | 6.7925  | 63000 | 0.3866   | 3.3583          |
+| 3.2875        | 6.9003  | 64000 | 0.3873   | 3.3529          |
+| 3.2082        | 7.0081  | 65000 | 0.3875   | 3.3548          |
+| 3.2291        | 7.1159  | 66000 | 0.3872   | 3.3594          |
+| 3.2144        | 7.2237  | 67000 | 0.3872   | 3.3567          |
+| 3.2273        | 7.3315  | 68000 | 0.3884   | 3.3511          |
+| 3.2405        | 7.4394  | 69000 | 0.3886   | 3.3481          |
+| 3.2308        | 7.5472  | 70000 | 0.3887   | 3.3429          |
+| 3.2202        | 7.6550  | 71000 | 0.3893   | 3.3402          |
+| 3.2477        | 7.7628  | 72000 | 0.3899   | 3.3355          |
+| 3.2472        | 7.8706  | 73000 | 0.3900   | 3.3337          |
+| 3.2422        | 7.9784  | 74000 | 0.3905   | 3.3273          |
+| 3.1478        | 8.0863  | 75000 | 0.3902   | 3.3339          |
+| 3.1733        | 8.1941  | 76000 | 0.3906   | 3.3336          |
+| 3.1655        | 8.3019  | 77000 | 0.3909   | 3.3311          |
+| 3.16          | 8.4097  | 78000 | 0.3911   | 3.3274          |
+| 3.1795        | 8.5175  | 79000 | 0.3913   | 3.3251          |
+| 3.1806        | 8.6253  | 80000 | 0.3918   | 3.3206          |
+| 3.1844        | 8.7332  | 81000 | 0.3922   | 3.3171          |
+| 3.1883        | 8.8410  | 82000 | 0.3926   | 3.3136          |
+| 3.164         | 8.9488  | 83000 | 0.3929   | 3.3117          |
+| 3.1285        | 9.0566  | 84000 | 0.3927   | 3.3139          |
+| 3.127         | 9.1644  | 85000 | 0.3929   | 3.3135          |
+| 3.109         | 9.2722  | 86000 | 0.3933   | 3.3108          |
+| 3.1161        | 9.3801  | 87000 | 0.3934   | 3.3099          |
+| 3.1228        | 9.4879  | 88000 | 0.3937   | 3.3063          |
+| 3.1352        | 9.5957  | 89000 | 0.3940   | 3.3041          |
+| 3.127         | 9.7035  | 90000 | 0.3942   | 3.3023          |
+| 3.1309        | 9.8113  | 91000 | 0.3945   | 3.3003          |
+| 3.1271        | 9.9191  | 92000 | 0.3946   | 3.2983          |
+| 3.2875        | 10.0270 | 93000 | 3.4554   | 0.3782          |
+| 3.3636        | 10.1348 | 94000 | 3.4681   | 0.3759          |
+| 3.3804        | 10.2426 | 95000 | 3.4619   | 0.3767          |
+### Framework versions
+- Transformers 4.47.0.dev0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.2
+- Tokenizers 0.20.1

base/100M__1208/all_results.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942401583029114,
+    "eval_loss": 3.3023483753204346,
+    "eval_runtime": 145.9968,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 123.366,
+    "eval_steps_per_second": 7.712,
+    "perplexity": 27.176384392621944,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07917592998303866,
+    "train_runtime": 1615.2207,
+    "train_samples": 296775,
+    "train_samples_per_second": 9186.825,
+    "train_steps_per_second": 287.112
+}

base/100M__1208/config.json ADDED Viewed

	@@ -0,0 +1,31 @@

+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 52000
+}

base/100M__1208/eval_results.json ADDED Viewed

	@@ -0,0 +1,10 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942401583029114,
+    "eval_loss": 3.3023483753204346,
+    "eval_runtime": 145.9968,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 123.366,
+    "eval_steps_per_second": 7.712,
+    "perplexity": 27.176384392621944
+}

base/100M__1208/generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.0.dev0"
+}

base/100M__1208/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1 @@


1	+ {}

base/100M__1208/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__1208/tokenizer_config.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__1208/train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07917592998303866,
+    "train_runtime": 1615.2207,
+    "train_samples": 296775,
+    "train_samples_per_second": 9186.825,
+    "train_steps_per_second": 287.112
+}

base/100M__1208/trainer_state.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__495/README.md ADDED Viewed

	@@ -0,0 +1,155 @@

+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: 100M__495
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+# 100M__495
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.4676
+- Accuracy: 0.3760
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 32
+- eval_batch_size: 16
+- seed: 495
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50
+- mixed_precision_training: Native AMP
+### Training results
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 5.1665        | 0.1078  | 1000  | 0.2206   | 5.0912          |
+| 4.6555        | 0.2156  | 2000  | 0.2607   | 4.6008          |
+| 4.3901        | 0.3235  | 3000  | 0.2916   | 4.2963          |
+| 4.2075        | 0.4313  | 4000  | 0.3082   | 4.1342          |
+| 4.0819        | 0.5391  | 5000  | 0.3186   | 4.0228          |
+| 4.0201        | 0.6469  | 6000  | 0.3257   | 3.9409          |
+| 3.9519        | 0.7547  | 7000  | 0.3312   | 3.8853          |
+| 3.8917        | 0.8625  | 8000  | 0.3359   | 3.8361          |
+| 3.8592        | 0.9704  | 9000  | 0.3404   | 3.7949          |
+| 3.7811        | 1.0782  | 10000 | 0.3437   | 3.7621          |
+| 3.7908        | 1.1860  | 11000 | 0.3460   | 3.7343          |
+| 3.7514        | 1.2938  | 12000 | 0.3482   | 3.7092          |
+| 3.7207        | 1.4016  | 13000 | 0.3513   | 3.6841          |
+| 3.7092        | 1.5094  | 14000 | 0.3533   | 3.6657          |
+| 3.6968        | 1.6173  | 15000 | 0.3550   | 3.6474          |
+| 3.6724        | 1.7251  | 16000 | 0.3571   | 3.6263          |
+| 3.6737        | 1.8329  | 17000 | 0.3582   | 3.6162          |
+| 3.6477        | 1.9407  | 18000 | 0.3598   | 3.5972          |
+| 3.5922        | 2.0485  | 19000 | 0.3611   | 3.5889          |
+| 3.5706        | 2.1563  | 20000 | 0.3621   | 3.5809          |
+| 3.5568        | 2.2642  | 21000 | 0.3631   | 3.5704          |
+| 3.5596        | 2.3720  | 22000 | 0.3645   | 3.5584          |
+| 3.5529        | 2.4798  | 23000 | 0.3655   | 3.5451          |
+| 3.5466        | 2.5876  | 24000 | 0.3665   | 3.5379          |
+| 3.555         | 2.6954  | 25000 | 0.3674   | 3.5254          |
+| 3.5597        | 2.8032  | 26000 | 0.3684   | 3.5180          |
+| 3.5422        | 2.9111  | 27000 | 0.3693   | 3.5087          |
+| 3.4422        | 3.0189  | 28000 | 0.3700   | 3.5041          |
+| 3.4594        | 3.1267  | 29000 | 0.3709   | 3.4994          |
+| 3.4767        | 3.2345  | 30000 | 0.3715   | 3.4932          |
+| 3.4629        | 3.3423  | 31000 | 0.3725   | 3.4884          |
+| 3.471         | 3.4501  | 32000 | 0.3728   | 3.4814          |
+| 3.4594        | 3.5580  | 33000 | 0.3735   | 3.4750          |
+| 3.4814        | 3.6658  | 34000 | 0.3739   | 3.4692          |
+| 3.4593        | 3.7736  | 35000 | 0.3747   | 3.4629          |
+| 3.4435        | 3.8814  | 36000 | 0.3756   | 3.4561          |
+| 3.4633        | 3.9892  | 37000 | 0.3759   | 3.4494          |
+| 3.381         | 4.0970  | 38000 | 0.3766   | 3.4523          |
+| 3.3899        | 4.2049  | 39000 | 0.3769   | 3.4458          |
+| 3.3917        | 4.3127  | 40000 | 0.3774   | 3.4431          |
+| 3.4098        | 4.4205  | 41000 | 0.3781   | 3.4349          |
+| 3.4073        | 4.5283  | 42000 | 0.3784   | 3.4332          |
+| 3.3993        | 4.6361  | 43000 | 0.3786   | 3.4270          |
+| 3.4026        | 4.7439  | 44000 | 0.3797   | 3.4206          |
+| 3.407         | 4.8518  | 45000 | 0.3798   | 3.4162          |
+| 3.3882        | 4.9596  | 46000 | 0.3807   | 3.4124          |
+| 3.3153        | 5.0674  | 47000 | 0.3809   | 3.4160          |
+| 3.3206        | 5.1752  | 48000 | 0.3811   | 3.4111          |
+| 3.3424        | 5.2830  | 49000 | 0.3814   | 3.4074          |
+| 3.3458        | 5.3908  | 50000 | 0.3819   | 3.4056          |
+| 3.3476        | 5.4987  | 51000 | 0.3825   | 3.4017          |
+| 3.3541        | 5.6065  | 52000 | 0.3827   | 3.3958          |
+| 3.347         | 5.7143  | 53000 | 0.3831   | 3.3918          |
+| 3.3347        | 5.8221  | 54000 | 0.3836   | 3.3862          |
+| 3.3565        | 5.9299  | 55000 | 0.3844   | 3.3812          |
+| 3.2551        | 6.0377  | 56000 | 0.3845   | 3.3851          |
+| 3.2716        | 6.1456  | 57000 | 0.3843   | 3.3840          |
+| 3.2895        | 6.2534  | 58000 | 0.3848   | 3.3813          |
+| 3.2759        | 6.3612  | 59000 | 0.3856   | 3.3767          |
+| 3.2909        | 6.4690  | 60000 | 0.3853   | 3.3733          |
+| 3.2835        | 6.5768  | 61000 | 0.3859   | 3.3691          |
+| 3.3046        | 6.6846  | 62000 | 0.3865   | 3.3650          |
+| 3.2794        | 6.7925  | 63000 | 0.3867   | 3.3618          |
+| 3.2831        | 6.9003  | 64000 | 0.3873   | 3.3556          |
+| 3.2134        | 7.0081  | 65000 | 0.3874   | 3.3601          |
+| 3.2066        | 7.1159  | 66000 | 0.3873   | 3.3606          |
+| 3.2313        | 7.2237  | 67000 | 0.3881   | 3.3570          |
+| 3.2319        | 7.3315  | 68000 | 0.3881   | 3.3540          |
+| 3.2282        | 7.4394  | 69000 | 0.3882   | 3.3505          |
+| 3.2386        | 7.5472  | 70000 | 0.3889   | 3.3455          |
+| 3.233         | 7.6550  | 71000 | 0.3891   | 3.3429          |
+| 3.2386        | 7.7628  | 72000 | 0.3898   | 3.3377          |
+| 3.247         | 7.8706  | 73000 | 0.3899   | 3.3356          |
+| 3.2571        | 7.9784  | 74000 | 0.3908   | 3.3312          |
+| 3.1695        | 8.0863  | 75000 | 0.3905   | 3.3377          |
+| 3.1903        | 8.1941  | 76000 | 0.3906   | 3.3351          |
+| 3.1892        | 8.3019  | 77000 | 0.3909   | 3.3317          |
+| 3.1795        | 8.4097  | 78000 | 0.3912   | 3.3293          |
+| 3.179         | 8.5175  | 79000 | 0.3916   | 3.3260          |
+| 3.2147        | 8.6253  | 80000 | 0.3919   | 3.3226          |
+| 3.1679        | 8.7332  | 81000 | 0.3922   | 3.3194          |
+| 3.1853        | 8.8410  | 82000 | 0.3925   | 3.3157          |
+| 3.1983        | 8.9488  | 83000 | 0.3929   | 3.3126          |
+| 3.1342        | 9.0566  | 84000 | 0.3928   | 3.3160          |
+| 3.1281        | 9.1644  | 85000 | 0.3930   | 3.3144          |
+| 3.129         | 9.2722  | 86000 | 0.3934   | 3.3116          |
+| 3.1315        | 9.3801  | 87000 | 0.3937   | 3.3096          |
+| 3.1295        | 9.4879  | 88000 | 0.3940   | 3.3074          |
+| 3.1314        | 9.5957  | 89000 | 0.3942   | 3.3044          |
+| 3.1393        | 9.7035  | 90000 | 0.3945   | 3.3022          |
+| 3.1482        | 9.8113  | 91000 | 0.3948   | 3.3000          |
+| 3.1491        | 9.9191  | 92000 | 0.3950   | 3.2984          |
+| 3.3064        | 10.0270 | 93000 | 3.4581   | 0.3773          |
+| 3.3807        | 10.1348 | 94000 | 3.4690   | 0.3764          |
+| 3.3886        | 10.2426 | 95000 | 3.4676   | 0.3760          |
+### Framework versions
+- Transformers 4.47.0.dev0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.2
+- Tokenizers 0.20.1

base/100M__495/all_results.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3945371066453084,
+    "eval_loss": 3.302194118499756,
+    "eval_runtime": 147.1746,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 122.378,
+    "eval_steps_per_second": 7.651,
+    "perplexity": 27.17219257328467,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07955799303556743,
+    "train_runtime": 1634.6435,
+    "train_samples": 296775,
+    "train_samples_per_second": 9077.667,
+    "train_steps_per_second": 283.701
+}

base/100M__495/config.json ADDED Viewed

	@@ -0,0 +1,31 @@

+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 52000
+}

base/100M__495/eval_results.json ADDED Viewed

	@@ -0,0 +1,10 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3945371066453084,
+    "eval_loss": 3.302194118499756,
+    "eval_runtime": 147.1746,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 122.378,
+    "eval_steps_per_second": 7.651,
+    "perplexity": 27.17219257328467
+}

base/100M__495/generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.0.dev0"
+}

base/100M__495/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1 @@


1	+ {}

base/100M__495/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__495/tokenizer_config.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__495/train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07955799303556743,
+    "train_runtime": 1634.6435,
+    "train_samples": 296775,
+    "train_samples_per_second": 9077.667,
+    "train_steps_per_second": 283.701
+}

base/100M__495/trainer_state.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__634/README.md ADDED Viewed

	@@ -0,0 +1,155 @@

+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: 100M__634
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+# 100M__634
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.4625
+- Accuracy: 0.3763
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 32
+- eval_batch_size: 16
+- seed: 634
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50
+- mixed_precision_training: Native AMP
+### Training results
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 5.0778        | 0.1078  | 1000  | 0.2270   | 5.0244          |
+| 4.5738        | 0.2156  | 2000  | 0.2719   | 4.5008          |
+| 4.309         | 0.3235  | 3000  | 0.2995   | 4.2284          |
+| 4.1524        | 0.4313  | 4000  | 0.3122   | 4.0876          |
+| 4.0714        | 0.5391  | 5000  | 0.3216   | 3.9901          |
+| 3.9631        | 0.6469  | 6000  | 0.3281   | 3.9180          |
+| 3.9285        | 0.7547  | 7000  | 0.3334   | 3.8616          |
+| 3.8676        | 0.8625  | 8000  | 0.3375   | 3.8167          |
+| 3.8648        | 0.9704  | 9000  | 0.3409   | 3.7798          |
+| 3.7661        | 1.0782  | 10000 | 0.3448   | 3.7474          |
+| 3.7415        | 1.1860  | 11000 | 0.3470   | 3.7234          |
+| 3.7241        | 1.2938  | 12000 | 0.3486   | 3.7014          |
+| 3.715         | 1.4016  | 13000 | 0.3512   | 3.6799          |
+| 3.6982        | 1.5094  | 14000 | 0.3534   | 3.6556          |
+| 3.6829        | 1.6173  | 15000 | 0.3553   | 3.6376          |
+| 3.6554        | 1.7251  | 16000 | 0.3569   | 3.6238          |
+| 3.6345        | 1.8329  | 17000 | 0.3583   | 3.6062          |
+| 3.6469        | 1.9407  | 18000 | 0.3599   | 3.5914          |
+| 3.5639        | 2.0485  | 19000 | 0.3609   | 3.5848          |
+| 3.5606        | 2.1563  | 20000 | 0.3622   | 3.5729          |
+| 3.5497        | 2.2642  | 21000 | 0.3636   | 3.5615          |
+| 3.5624        | 2.3720  | 22000 | 0.3644   | 3.5510          |
+| 3.5459        | 2.4798  | 23000 | 0.3655   | 3.5396          |
+| 3.5482        | 2.5876  | 24000 | 0.3666   | 3.5318          |
+| 3.5542        | 2.6954  | 25000 | 0.3674   | 3.5221          |
+| 3.5342        | 2.8032  | 26000 | 0.3682   | 3.5146          |
+| 3.5381        | 2.9111  | 27000 | 0.3693   | 3.5050          |
+| 3.4278        | 3.0189  | 28000 | 0.3700   | 3.5020          |
+| 3.4513        | 3.1267  | 29000 | 0.3708   | 3.4942          |
+| 3.4627        | 3.2345  | 30000 | 0.3713   | 3.4898          |
+| 3.4463        | 3.3423  | 31000 | 0.3720   | 3.4832          |
+| 3.461         | 3.4501  | 32000 | 0.3727   | 3.4776          |
+| 3.4454        | 3.5580  | 33000 | 0.3733   | 3.4710          |
+| 3.4627        | 3.6658  | 34000 | 0.3741   | 3.4632          |
+| 3.444         | 3.7736  | 35000 | 0.3743   | 3.4588          |
+| 3.4513        | 3.8814  | 36000 | 0.3752   | 3.4522          |
+| 3.4411        | 3.9892  | 37000 | 0.3758   | 3.4460          |
+| 3.3781        | 4.0970  | 38000 | 0.3764   | 3.4480          |
+| 3.3767        | 4.2049  | 39000 | 0.3767   | 3.4423          |
+| 3.3947        | 4.3127  | 40000 | 0.3773   | 3.4385          |
+| 3.389         | 4.4205  | 41000 | 0.3776   | 3.4339          |
+| 3.3849        | 4.5283  | 42000 | 0.3786   | 3.4289          |
+| 3.4041        | 4.6361  | 43000 | 0.3791   | 3.4244          |
+| 3.3693        | 4.7439  | 44000 | 0.3796   | 3.4180          |
+| 3.3953        | 4.8518  | 45000 | 0.3799   | 3.4133          |
+| 3.3841        | 4.9596  | 46000 | 0.3805   | 3.4098          |
+| 3.3078        | 5.0674  | 47000 | 0.3803   | 3.4134          |
+| 3.315         | 5.1752  | 48000 | 0.3810   | 3.4103          |
+| 3.3108        | 5.2830  | 49000 | 0.3809   | 3.4087          |
+| 3.3356        | 5.3908  | 50000 | 0.3818   | 3.4030          |
+| 3.3193        | 5.4987  | 51000 | 0.3824   | 3.3966          |
+| 3.3407        | 5.6065  | 52000 | 0.3826   | 3.3946          |
+| 3.348         | 5.7143  | 53000 | 0.3831   | 3.3898          |
+| 3.3288        | 5.8221  | 54000 | 0.3834   | 3.3838          |
+| 3.3283        | 5.9299  | 55000 | 0.3838   | 3.3800          |
+| 3.2613        | 6.0377  | 56000 | 0.3840   | 3.3837          |
+| 3.2509        | 6.1456  | 57000 | 0.3840   | 3.3834          |
+| 3.2539        | 6.2534  | 58000 | 0.3846   | 3.3802          |
+| 3.2851        | 6.3612  | 59000 | 0.3851   | 3.3774          |
+| 3.2878        | 6.4690  | 60000 | 0.3853   | 3.3725          |
+| 3.2817        | 6.5768  | 61000 | 0.3855   | 3.3693          |
+| 3.2722        | 6.6846  | 62000 | 0.3863   | 3.3639          |
+| 3.2773        | 6.7925  | 63000 | 0.3867   | 3.3609          |
+| 3.2948        | 6.9003  | 64000 | 0.3872   | 3.3567          |
+| 3.1985        | 7.0081  | 65000 | 0.3872   | 3.3584          |
+| 3.2183        | 7.1159  | 66000 | 0.3874   | 3.3596          |
+| 3.2204        | 7.2237  | 67000 | 0.3878   | 3.3558          |
+| 3.2237        | 7.3315  | 68000 | 0.3881   | 3.3549          |
+| 3.2464        | 7.4394  | 69000 | 0.3881   | 3.3498          |
+| 3.2299        | 7.5472  | 70000 | 0.3887   | 3.3457          |
+| 3.2273        | 7.6550  | 71000 | 0.3889   | 3.3421          |
+| 3.2389        | 7.7628  | 72000 | 0.3894   | 3.3363          |
+| 3.2224        | 7.8706  | 73000 | 0.3899   | 3.3338          |
+| 3.2305        | 7.9784  | 74000 | 0.3902   | 3.3320          |
+| 3.1599        | 8.0863  | 75000 | 0.3901   | 3.3381          |
+| 3.1664        | 8.1941  | 76000 | 0.3903   | 3.3350          |
+| 3.1774        | 8.3019  | 77000 | 0.3903   | 3.3318          |
+| 3.1677        | 8.4097  | 78000 | 0.3909   | 3.3281          |
+| 3.1821        | 8.5175  | 79000 | 0.3913   | 3.3258          |
+| 3.1805        | 8.6253  | 80000 | 0.3917   | 3.3235          |
+| 3.1932        | 8.7332  | 81000 | 0.3920   | 3.3191          |
+| 3.1771        | 8.8410  | 82000 | 0.3924   | 3.3148          |
+| 3.1927        | 8.9488  | 83000 | 0.3927   | 3.3131          |
+| 3.1235        | 9.0566  | 84000 | 0.3926   | 3.3163          |
+| 3.1221        | 9.1644  | 85000 | 0.3929   | 3.3138          |
+| 3.1312        | 9.2722  | 86000 | 0.3931   | 3.3123          |
+| 3.1204        | 9.3801  | 87000 | 0.3934   | 3.3111          |
+| 3.1289        | 9.4879  | 88000 | 0.3939   | 3.3072          |
+| 3.1452        | 9.5957  | 89000 | 0.3940   | 3.3056          |
+| 3.1301        | 9.7035  | 90000 | 0.3943   | 3.3030          |
+| 3.1314        | 9.8113  | 91000 | 0.3943   | 3.3021          |
+| 3.1061        | 9.9191  | 92000 | 0.3946   | 3.3004          |
+| 3.2987        | 10.0270 | 93000 | 3.4598   | 0.3772          |
+| 3.3802        | 10.1348 | 94000 | 3.4705   | 0.3756          |
+| 3.3831        | 10.2426 | 95000 | 3.4625   | 0.3763          |
+### Framework versions
+- Transformers 4.47.0.dev0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.2
+- Tokenizers 0.20.1

base/100M__634/all_results.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942879655735498,
+    "eval_loss": 3.303001642227173,
+    "eval_runtime": 145.5138,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 123.775,
+    "eval_steps_per_second": 7.738,
+    "perplexity": 27.19414362531837,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07928502052708676,
+    "train_runtime": 1615.2002,
+    "train_samples": 296775,
+    "train_samples_per_second": 9186.942,
+    "train_steps_per_second": 287.116
+}

base/100M__634/config.json ADDED Viewed

	@@ -0,0 +1,31 @@

+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 52000
+}

base/100M__634/eval_results.json ADDED Viewed

	@@ -0,0 +1,10 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942879655735498,
+    "eval_loss": 3.303001642227173,
+    "eval_runtime": 145.5138,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 123.775,
+    "eval_steps_per_second": 7.738,
+    "perplexity": 27.19414362531837
+}

base/100M__634/generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.0.dev0"
+}

base/100M__634/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1 @@


1	+ {}

base/100M__634/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__634/tokenizer_config.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__634/train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07928502052708676,
+    "train_runtime": 1615.2002,
+    "train_samples": 296775,
+    "train_samples_per_second": 9186.942,
+    "train_steps_per_second": 287.116
+}

base/100M__634/trainer_state.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__6910/README.md ADDED Viewed

	@@ -0,0 +1,155 @@

+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: 100M__6910
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+# 100M__6910
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.4641
+- Accuracy: 0.3759
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 32
+- eval_batch_size: 16
+- seed: 6910
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50
+- mixed_precision_training: Native AMP
+### Training results
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 5.1167        | 0.1078  | 1000  | 0.2265   | 5.0316          |
+| 4.5939        | 0.2156  | 2000  | 0.2701   | 4.5123          |
+| 4.3174        | 0.3235  | 3000  | 0.2980   | 4.2394          |
+| 4.1616        | 0.4313  | 4000  | 0.3117   | 4.0950          |
+| 4.0631        | 0.5391  | 5000  | 0.3210   | 3.9943          |
+| 4.0063        | 0.6469  | 6000  | 0.3278   | 3.9201          |
+| 3.9296        | 0.7547  | 7000  | 0.3329   | 3.8656          |
+| 3.8882        | 0.8625  | 8000  | 0.3372   | 3.8190          |
+| 3.8432        | 0.9704  | 9000  | 0.3407   | 3.7808          |
+| 3.7782        | 1.0782  | 10000 | 0.3441   | 3.7569          |
+| 3.7405        | 1.1860  | 11000 | 0.3471   | 3.7269          |
+| 3.7342        | 1.2938  | 12000 | 0.3493   | 3.7020          |
+| 3.7098        | 1.4016  | 13000 | 0.3514   | 3.6816          |
+| 3.7016        | 1.5094  | 14000 | 0.3536   | 3.6587          |
+| 3.6781        | 1.6173  | 15000 | 0.3555   | 3.6408          |
+| 3.6798        | 1.7251  | 16000 | 0.3569   | 3.6209          |
+| 3.6639        | 1.8329  | 17000 | 0.3586   | 3.6067          |
+| 3.6461        | 1.9407  | 18000 | 0.3602   | 3.5905          |
+| 3.5609        | 2.0485  | 19000 | 0.3615   | 3.5818          |
+| 3.5583        | 2.1563  | 20000 | 0.3628   | 3.5729          |
+| 3.5592        | 2.2642  | 21000 | 0.3634   | 3.5644          |
+| 3.5447        | 2.3720  | 22000 | 0.3647   | 3.5518          |
+| 3.5527        | 2.4798  | 23000 | 0.3656   | 3.5423          |
+| 3.541         | 2.5876  | 24000 | 0.3668   | 3.5317          |
+| 3.5277        | 2.6954  | 25000 | 0.3674   | 3.5235          |
+| 3.5395        | 2.8032  | 26000 | 0.3687   | 3.5142          |
+| 3.522         | 2.9111  | 27000 | 0.3694   | 3.5048          |
+| 3.4455        | 3.0189  | 28000 | 0.3705   | 3.5026          |
+| 3.4399        | 3.1267  | 29000 | 0.3706   | 3.4956          |
+| 3.4688        | 3.2345  | 30000 | 0.3718   | 3.4911          |
+| 3.4748        | 3.3423  | 31000 | 0.3726   | 3.4835          |
+| 3.4633        | 3.4501  | 32000 | 0.3733   | 3.4767          |
+| 3.4541        | 3.5580  | 33000 | 0.3735   | 3.4712          |
+| 3.4513        | 3.6658  | 34000 | 0.3745   | 3.4640          |
+| 3.4707        | 3.7736  | 35000 | 0.3750   | 3.4572          |
+| 3.4347        | 3.8814  | 36000 | 0.3755   | 3.4530          |
+| 3.417         | 3.9892  | 37000 | 0.3759   | 3.4462          |
+| 3.3602        | 4.0970  | 38000 | 0.3764   | 3.4484          |
+| 3.384         | 4.2049  | 39000 | 0.3770   | 3.4457          |
+| 3.4014        | 4.3127  | 40000 | 0.3773   | 3.4402          |
+| 3.4152        | 4.4205  | 41000 | 0.3779   | 3.4369          |
+| 3.3867        | 4.5283  | 42000 | 0.3787   | 3.4306          |
+| 3.4083        | 4.6361  | 43000 | 0.3791   | 3.4263          |
+| 3.3929        | 4.7439  | 44000 | 0.3794   | 3.4202          |
+| 3.389         | 4.8518  | 45000 | 0.3803   | 3.4123          |
+| 3.3876        | 4.9596  | 46000 | 0.3805   | 3.4127          |
+| 3.3161        | 5.0674  | 47000 | 0.3810   | 3.4123          |
+| 3.3084        | 5.1752  | 48000 | 0.3809   | 3.4124          |
+| 3.329         | 5.2830  | 49000 | 0.3815   | 3.4083          |
+| 3.3415        | 5.3908  | 50000 | 0.3817   | 3.4022          |
+| 3.3172        | 5.4987  | 51000 | 0.3822   | 3.3990          |
+| 3.3287        | 5.6065  | 52000 | 0.3826   | 3.3959          |
+| 3.3407        | 5.7143  | 53000 | 0.3830   | 3.3904          |
+| 3.3252        | 5.8221  | 54000 | 0.3834   | 3.3858          |
+| 3.3267        | 5.9299  | 55000 | 0.3839   | 3.3821          |
+| 3.2385        | 6.0377  | 56000 | 0.3840   | 3.3859          |
+| 3.2695        | 6.1456  | 57000 | 0.3841   | 3.3841          |
+| 3.2792        | 6.2534  | 58000 | 0.3846   | 3.3813          |
+| 3.2941        | 6.3612  | 59000 | 0.3850   | 3.3767          |
+| 3.2853        | 6.4690  | 60000 | 0.3855   | 3.3732          |
+| 3.2766        | 6.5768  | 61000 | 0.3860   | 3.3684          |
+| 3.2783        | 6.6846  | 62000 | 0.3860   | 3.3660          |
+| 3.2875        | 6.7925  | 63000 | 0.3866   | 3.3614          |
+| 3.3016        | 6.9003  | 64000 | 0.3870   | 3.3557          |
+| 3.2097        | 7.0081  | 65000 | 0.3869   | 3.3597          |
+| 3.2227        | 7.1159  | 66000 | 0.3876   | 3.3602          |
+| 3.2331        | 7.2237  | 67000 | 0.3875   | 3.3562          |
+| 3.2154        | 7.3315  | 68000 | 0.3880   | 3.3542          |
+| 3.2493        | 7.4394  | 69000 | 0.3881   | 3.3500          |
+| 3.2359        | 7.5472  | 70000 | 0.3887   | 3.3460          |
+| 3.236         | 7.6550  | 71000 | 0.3892   | 3.3415          |
+| 3.2325        | 7.7628  | 72000 | 0.3895   | 3.3391          |
+| 3.2321        | 7.8706  | 73000 | 0.3898   | 3.3348          |
+| 3.2455        | 7.9784  | 74000 | 0.3900   | 3.3332          |
+| 3.1591        | 8.0863  | 75000 | 0.3900   | 3.3374          |
+| 3.1871        | 8.1941  | 76000 | 0.3901   | 3.3381          |
+| 3.1747        | 8.3019  | 77000 | 0.3905   | 3.3329          |
+| 3.1932        | 8.4097  | 78000 | 0.3909   | 3.3302          |
+| 3.1805        | 8.5175  | 79000 | 0.3911   | 3.3280          |
+| 3.1777        | 8.6253  | 80000 | 0.3917   | 3.3246          |
+| 3.1899        | 8.7332  | 81000 | 0.3921   | 3.3193          |
+| 3.196         | 8.8410  | 82000 | 0.3924   | 3.3160          |
+| 3.1743        | 8.9488  | 83000 | 0.3926   | 3.3144          |
+| 3.1092        | 9.0566  | 84000 | 0.3926   | 3.3175          |
+| 3.1468        | 9.1644  | 85000 | 0.3929   | 3.3152          |
+| 3.1336        | 9.2722  | 86000 | 0.3930   | 3.3148          |
+| 3.1193        | 9.3801  | 87000 | 0.3933   | 3.3122          |
+| 3.1383        | 9.4879  | 88000 | 0.3938   | 3.3087          |
+| 3.13          | 9.5957  | 89000 | 0.3938   | 3.3068          |
+| 3.135         | 9.7035  | 90000 | 0.3942   | 3.3042          |
+| 3.1324        | 9.8113  | 91000 | 0.3944   | 3.3028          |
+| 3.1419        | 9.9191  | 92000 | 0.3946   | 3.3012          |
+| 3.2895        | 10.0270 | 93000 | 3.4581   | 0.3781          |
+| 3.3641        | 10.1348 | 94000 | 3.4706   | 0.3758          |
+| 3.3727        | 10.2426 | 95000 | 3.4641   | 0.3759          |
+### Framework versions
+- Transformers 4.47.0.dev0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.2
+- Tokenizers 0.20.1

base/100M__6910/all_results.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.39419332890826336,
+    "eval_loss": 3.304239273071289,
+    "eval_runtime": 100.8226,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 178.641,
+    "eval_steps_per_second": 11.168,
+    "perplexity": 27.22782077188737,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07937320893940172,
+    "train_runtime": 864.8237,
+    "train_samples": 296775,
+    "train_samples_per_second": 17158.121,
+    "train_steps_per_second": 536.236
+}

base/100M__6910/config.json ADDED Viewed

	@@ -0,0 +1,31 @@

+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 52000
+}

base/100M__6910/eval_results.json ADDED Viewed

	@@ -0,0 +1,10 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.39419332890826336,
+    "eval_loss": 3.304239273071289,
+    "eval_runtime": 100.8226,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 178.641,
+    "eval_steps_per_second": 11.168,
+    "perplexity": 27.22782077188737
+}

base/100M__6910/generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.0.dev0"
+}

base/100M__6910/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1 @@


1	+ {}

base/100M__6910/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__6910/tokenizer_config.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__6910/train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07937320893940172,
+    "train_runtime": 864.8237,
+    "train_samples": 296775,
+    "train_samples_per_second": 17158.121,
+    "train_steps_per_second": 536.236
+}

base/100M__6910/trainer_state.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__8397/README.md ADDED Viewed

	@@ -0,0 +1,155 @@

+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: 100M__8397
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+# 100M__8397
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.4643
+- Accuracy: 0.3759
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 32
+- eval_batch_size: 16
+- seed: 8397
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50
+- mixed_precision_training: Native AMP
+### Training results
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 5.0934        | 0.1078  | 1000  | 0.2275   | 5.0225          |
+| 4.5937        | 0.2156  | 2000  | 0.2702   | 4.5165          |
+| 4.3092        | 0.3235  | 3000  | 0.2978   | 4.2461          |
+| 4.1586        | 0.4313  | 4000  | 0.3127   | 4.0885          |
+| 4.0492        | 0.5391  | 5000  | 0.3212   | 3.9914          |
+| 3.9962        | 0.6469  | 6000  | 0.3279   | 3.9193          |
+| 3.9219        | 0.7547  | 7000  | 0.3335   | 3.8632          |
+| 3.8566        | 0.8625  | 8000  | 0.3377   | 3.8155          |
+| 3.856         | 0.9704  | 9000  | 0.3408   | 3.7813          |
+| 3.7649        | 1.0782  | 10000 | 0.3444   | 3.7518          |
+| 3.769         | 1.1860  | 11000 | 0.3474   | 3.7232          |
+| 3.7288        | 1.2938  | 12000 | 0.3493   | 3.6971          |
+| 3.7086        | 1.4016  | 13000 | 0.3515   | 3.6748          |
+| 3.703         | 1.5094  | 14000 | 0.3535   | 3.6584          |
+| 3.664         | 1.6173  | 15000 | 0.3555   | 3.6374          |
+| 3.6639        | 1.7251  | 16000 | 0.3572   | 3.6206          |
+| 3.6383        | 1.8329  | 17000 | 0.3587   | 3.6051          |
+| 3.6321        | 1.9407  | 18000 | 0.3600   | 3.5912          |
+| 3.5672        | 2.0485  | 19000 | 0.3615   | 3.5821          |
+| 3.5754        | 2.1563  | 20000 | 0.3624   | 3.5733          |
+| 3.5637        | 2.2642  | 21000 | 0.3637   | 3.5613          |
+| 3.5706        | 2.3720  | 22000 | 0.3646   | 3.5504          |
+| 3.5408        | 2.4798  | 23000 | 0.3660   | 3.5402          |
+| 3.532         | 2.5876  | 24000 | 0.3669   | 3.5312          |
+| 3.527         | 2.6954  | 25000 | 0.3673   | 3.5218          |
+| 3.5333        | 2.8032  | 26000 | 0.3685   | 3.5120          |
+| 3.5442        | 2.9111  | 27000 | 0.3696   | 3.5054          |
+| 3.4451        | 3.0189  | 28000 | 0.3700   | 3.5012          |
+| 3.4295        | 3.1267  | 29000 | 0.3709   | 3.4969          |
+| 3.4506        | 3.2345  | 30000 | 0.3713   | 3.4902          |
+| 3.4582        | 3.3423  | 31000 | 0.3719   | 3.4849          |
+| 3.4597        | 3.4501  | 32000 | 0.3733   | 3.4757          |
+| 3.4639        | 3.5580  | 33000 | 0.3736   | 3.4722          |
+| 3.4602        | 3.6658  | 34000 | 0.3735   | 3.4653          |
+| 3.4613        | 3.7736  | 35000 | 0.3749   | 3.4591          |
+| 3.4517        | 3.8814  | 36000 | 0.3755   | 3.4511          |
+| 3.4488        | 3.9892  | 37000 | 0.3761   | 3.4457          |
+| 3.3644        | 4.0970  | 38000 | 0.3764   | 3.4490          |
+| 3.3636        | 4.2049  | 39000 | 0.3773   | 3.4452          |
+| 3.3899        | 4.3127  | 40000 | 0.3775   | 3.4419          |
+| 3.3969        | 4.4205  | 41000 | 0.3782   | 3.4341          |
+| 3.3848        | 4.5283  | 42000 | 0.3786   | 3.4308          |
+| 3.3952        | 4.6361  | 43000 | 0.3788   | 3.4244          |
+| 3.3875        | 4.7439  | 44000 | 0.3793   | 3.4211          |
+| 3.4071        | 4.8518  | 45000 | 0.3799   | 3.4152          |
+| 3.3742        | 4.9596  | 46000 | 0.3803   | 3.4095          |
+| 3.3158        | 5.0674  | 47000 | 0.3807   | 3.4110          |
+| 3.3119        | 5.1752  | 48000 | 0.3812   | 3.4111          |
+| 3.3434        | 5.2830  | 49000 | 0.3813   | 3.4074          |
+| 3.3422        | 5.3908  | 50000 | 0.3820   | 3.4001          |
+| 3.3445        | 5.4987  | 51000 | 0.3820   | 3.3994          |
+| 3.3169        | 5.6065  | 52000 | 0.3826   | 3.3929          |
+| 3.333         | 5.7143  | 53000 | 0.3832   | 3.3878          |
+| 3.331         | 5.8221  | 54000 | 0.3835   | 3.3844          |
+| 3.3342        | 5.9299  | 55000 | 0.3840   | 3.3803          |
+| 3.2353        | 6.0377  | 56000 | 0.3840   | 3.3844          |
+| 3.2567        | 6.1456  | 57000 | 0.3843   | 3.3837          |
+| 3.2751        | 6.2534  | 58000 | 0.3847   | 3.3811          |
+| 3.2877        | 6.3612  | 59000 | 0.3850   | 3.3768          |
+| 3.2882        | 6.4690  | 60000 | 0.3854   | 3.3732          |
+| 3.2801        | 6.5768  | 61000 | 0.3860   | 3.3683          |
+| 3.2991        | 6.6846  | 62000 | 0.3863   | 3.3624          |
+| 3.2791        | 6.7925  | 63000 | 0.3866   | 3.3601          |
+| 3.288         | 6.9003  | 64000 | 0.3871   | 3.3546          |
+| 3.1829        | 7.0081  | 65000 | 0.3871   | 3.3573          |
+| 3.2234        | 7.1159  | 66000 | 0.3873   | 3.3607          |
+| 3.2282        | 7.2237  | 67000 | 0.3876   | 3.3572          |
+| 3.2267        | 7.3315  | 68000 | 0.3878   | 3.3537          |
+| 3.218         | 7.4394  | 69000 | 0.3883   | 3.3486          |
+| 3.2235        | 7.5472  | 70000 | 0.3885   | 3.3458          |
+| 3.2512        | 7.6550  | 71000 | 0.3889   | 3.3421          |
+| 3.2503        | 7.7628  | 72000 | 0.3895   | 3.3384          |
+| 3.2303        | 7.8706  | 73000 | 0.3901   | 3.3343          |
+| 3.2506        | 7.9784  | 74000 | 0.3901   | 3.3303          |
+| 3.155         | 8.0863  | 75000 | 0.3902   | 3.3376          |
+| 3.1552        | 8.1941  | 76000 | 0.3902   | 3.3350          |
+| 3.1737        | 8.3019  | 77000 | 0.3907   | 3.3308          |
+| 3.167         | 8.4097  | 78000 | 0.3908   | 3.3294          |
+| 3.1798        | 8.5175  | 79000 | 0.3912   | 3.3247          |
+| 3.1957        | 8.6253  | 80000 | 0.3918   | 3.3219          |
+| 3.1876        | 8.7332  | 81000 | 0.3921   | 3.3189          |
+| 3.1825        | 8.8410  | 82000 | 0.3922   | 3.3150          |
+| 3.1641        | 8.9488  | 83000 | 0.3927   | 3.3115          |
+| 3.1278        | 9.0566  | 84000 | 0.3927   | 3.3139          |
+| 3.1192        | 9.1644  | 85000 | 0.3928   | 3.3138          |
+| 3.1443        | 9.2722  | 86000 | 0.3932   | 3.3118          |
+| 3.1211        | 9.3801  | 87000 | 0.3934   | 3.3095          |
+| 3.1296        | 9.4879  | 88000 | 0.3936   | 3.3073          |
+| 3.1073        | 9.5957  | 89000 | 0.3939   | 3.3058          |
+| 3.1362        | 9.7035  | 90000 | 0.3942   | 3.3025          |
+| 3.1261        | 9.8113  | 91000 | 0.3944   | 3.3008          |
+| 3.1183        | 9.9191  | 92000 | 0.3945   | 3.2993          |
+| 3.3293        | 10.0270 | 93000 | 3.4597   | 0.3782          |
+| 3.364         | 10.1348 | 94000 | 3.4676   | 0.3759          |
+| 3.3881        | 10.2426 | 95000 | 3.4643   | 0.3759          |
+### Framework versions
+- Transformers 4.47.0.dev0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.2
+- Tokenizers 0.20.1

base/100M__8397/all_results.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942060412961377,
+    "eval_loss": 3.302459716796875,
+    "eval_runtime": 97.9886,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 183.807,
+    "eval_steps_per_second": 11.491,
+    "perplexity": 27.179410419842576,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07933711596037211,
+    "train_runtime": 850.9952,
+    "train_samples": 296775,
+    "train_samples_per_second": 17436.937,
+    "train_steps_per_second": 544.95
+}

base/100M__8397/config.json ADDED Viewed

	@@ -0,0 +1,31 @@

+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 52000
+}

base/100M__8397/eval_results.json ADDED Viewed

	@@ -0,0 +1,10 @@

+{
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942060412961377,
+    "eval_loss": 3.302459716796875,
+    "eval_runtime": 97.9886,
+    "eval_samples": 18011,
+    "eval_samples_per_second": 183.807,
+    "eval_steps_per_second": 11.491,
+    "perplexity": 27.179410419842576
+}

base/100M__8397/generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.0.dev0"
+}

base/100M__8397/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1 @@


1	+ {}

base/100M__8397/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__8397/tokenizer_config.json ADDED Viewed

The diff for this file is too large to render. See raw diff

base/100M__8397/train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07933711596037211,
+    "train_runtime": 850.9952,
+    "train_samples": 296775,
+    "train_samples_per_second": 17436.937,
+    "train_steps_per_second": 544.95
+}

base/100M__8397/trainer_state.json ADDED Viewed

The diff for this file is too large to render. See raw diff