fabikru commited on
Commit
f9b6f16
·
verified ·
1 Parent(s): c9caeb2

model_111M_medium_ds_masking_30_predicted_hparams

Browse files
Files changed (4) hide show
  1. README.md +34 -48
  2. config.json +4 -4
  3. model.safetensors +2 -2
  4. training_args.bin +1 -1
README.md CHANGED
@@ -16,8 +16,8 @@ should probably proofread and complete it, then remove this comment. -->
16
 
17
  This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
- - Loss: 0.1735
20
- - Accuracy: 0.9399
21
 
22
  ## Model description
23
 
@@ -36,12 +36,12 @@ More information needed
36
  ### Training hyperparameters
37
 
38
  The following hyperparameters were used during training:
39
- - learning_rate: 0.032227
40
- - train_batch_size: 1028
41
- - eval_batch_size: 1028
42
  - seed: 42
43
- - gradient_accumulation_steps: 4
44
- - total_train_batch_size: 4112
45
  - optimizer: Use OptimizerNames.SCHEDULE_FREE_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
46
  - lr_scheduler_type: constant
47
  - lr_scheduler_warmup_steps: 1000
@@ -50,47 +50,33 @@ The following hyperparameters were used during training:
50
 
51
  ### Training results
52
 
53
- | Training Loss | Epoch | Step | Validation Loss | Accuracy |
54
- |:-------------:|:------:|:----:|:---------------:|:--------:|
55
- | No log | 0 | 0 | 4.4310 | 0.0041 |
56
- | No log | 0.0043 | 121 | 0.7897 | 0.7545 |
57
- | No log | 0.0087 | 242 | 0.5334 | 0.8269 |
58
- | No log | 0.0130 | 363 | 0.4359 | 0.8565 |
59
- | No log | 0.0174 | 484 | 0.3745 | 0.8758 |
60
- | 3.2501 | 0.0217 | 605 | 0.3285 | 0.8900 |
61
- | 3.2501 | 0.0261 | 726 | 0.2951 | 0.9012 |
62
- | 3.2501 | 0.0304 | 847 | 0.2984 | 0.9001 |
63
- | 3.2501 | 0.0348 | 968 | 0.2718 | 0.9084 |
64
- | 1.4724 | 0.0391 | 1089 | 0.2632 | 0.9108 |
65
- | 1.4724 | 0.0435 | 1210 | 0.2468 | 0.9164 |
66
- | 1.4724 | 0.0478 | 1331 | 0.2449 | 0.9167 |
67
- | 1.4724 | 0.0522 | 1452 | 0.2366 | 0.9196 |
68
- | 1.0732 | 0.0565 | 1573 | 0.2255 | 0.9230 |
69
- | 1.0732 | 0.0609 | 1694 | 0.2225 | 0.9241 |
70
- | 1.0732 | 0.0652 | 1815 | 0.2176 | 0.9255 |
71
- | 1.0732 | 0.0696 | 1936 | 0.2115 | 0.9276 |
72
- | 0.9202 | 0.0739 | 2057 | 0.2075 | 0.9287 |
73
- | 0.9202 | 0.0783 | 2178 | 0.2061 | 0.9293 |
74
- | 0.9202 | 0.0826 | 2299 | 0.2024 | 0.9304 |
75
- | 0.9202 | 0.0870 | 2420 | 0.2020 | 0.9311 |
76
- | 0.8715 | 0.0913 | 2541 | 0.1986 | 0.9321 |
77
- | 0.8715 | 0.0957 | 2662 | 0.1973 | 0.9324 |
78
- | 0.8715 | 0.1000 | 2783 | 0.1959 | 0.9328 |
79
- | 0.8715 | 0.1044 | 2904 | 0.1932 | 0.9337 |
80
- | 0.8152 | 0.1087 | 3025 | 0.1880 | 0.9354 |
81
- | 0.8152 | 0.1131 | 3146 | 0.1883 | 0.9353 |
82
- | 0.8152 | 0.1174 | 3267 | 0.1856 | 0.9360 |
83
- | 0.8152 | 0.1218 | 3388 | 0.1855 | 0.9362 |
84
- | 0.7847 | 0.1261 | 3509 | 0.1837 | 0.9369 |
85
- | 0.7847 | 0.1305 | 3630 | 0.1826 | 0.9373 |
86
- | 0.7847 | 0.1348 | 3751 | 0.1813 | 0.9375 |
87
- | 0.7847 | 0.1392 | 3872 | 0.1809 | 0.9373 |
88
- | 0.7847 | 0.1435 | 3993 | 0.1790 | 0.9381 |
89
- | 0.7671 | 0.1479 | 4114 | 0.1769 | 0.9387 |
90
- | 0.7671 | 0.1522 | 4235 | 0.1778 | 0.9384 |
91
- | 0.7671 | 0.1566 | 4356 | 0.1734 | 0.9399 |
92
- | 0.7671 | 0.1609 | 4477 | 0.1740 | 0.9397 |
93
- | 0.741 | 0.1653 | 4598 | 0.1730 | 0.9405 |
94
 
95
 
96
  ### Framework versions
 
16
 
17
  This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
+ - Loss: 0.1486
20
+ - Accuracy: 0.9466
21
 
22
  ## Model description
23
 
 
36
  ### Training hyperparameters
37
 
38
  The following hyperparameters were used during training:
39
+ - learning_rate: 0.001033
40
+ - train_batch_size: 256
41
+ - eval_batch_size: 256
42
  - seed: 42
43
+ - gradient_accumulation_steps: 2
44
+ - total_train_batch_size: 512
45
  - optimizer: Use OptimizerNames.SCHEDULE_FREE_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
46
  - lr_scheduler_type: constant
47
  - lr_scheduler_warmup_steps: 1000
 
50
 
51
  ### Training results
52
 
53
+ | Training Loss | Epoch | Step | Validation Loss | Accuracy |
54
+ |:-------------:|:------:|:-----:|:---------------:|:--------:|
55
+ | No log | 0 | 0 | 4.4412 | 0.0013 |
56
+ | 2.1016 | 0.2150 | 976 | 0.3776 | 0.8695 |
57
+ | 0.725 | 0.4300 | 1952 | 0.2862 | 0.8996 |
58
+ | 0.5584 | 0.6450 | 2928 | 0.2479 | 0.9125 |
59
+ | 0.4968 | 0.8600 | 3904 | 0.2272 | 0.9196 |
60
+ | 0.4598 | 1.0749 | 4880 | 0.2133 | 0.9242 |
61
+ | 0.4333 | 1.2899 | 5856 | 0.2031 | 0.9277 |
62
+ | 0.4153 | 1.5049 | 6832 | 0.1958 | 0.9301 |
63
+ | 0.4 | 1.7199 | 7808 | 0.1906 | 0.9319 |
64
+ | 0.3886 | 1.9349 | 8784 | 0.1837 | 0.9345 |
65
+ | 0.3775 | 2.1498 | 9760 | 0.1789 | 0.9358 |
66
+ | 0.3684 | 2.3648 | 10736 | 0.1767 | 0.9369 |
67
+ | 0.3596 | 2.5798 | 11712 | 0.1713 | 0.9386 |
68
+ | 0.3547 | 2.7948 | 12688 | 0.1687 | 0.9394 |
69
+ | 0.3458 | 3.0097 | 13664 | 0.1662 | 0.9401 |
70
+ | 0.34 | 3.2247 | 14640 | 0.1638 | 0.9413 |
71
+ | 0.3359 | 3.4397 | 15616 | 0.1623 | 0.9418 |
72
+ | 0.3302 | 3.6547 | 16592 | 0.1601 | 0.9424 |
73
+ | 0.3266 | 3.8697 | 17568 | 0.1576 | 0.9434 |
74
+ | 0.3224 | 4.0846 | 18544 | 0.1547 | 0.9446 |
75
+ | 0.3191 | 4.2996 | 19520 | 0.1541 | 0.9444 |
76
+ | 0.3149 | 4.5146 | 20496 | 0.1518 | 0.9453 |
77
+ | 0.3103 | 4.7296 | 21472 | 0.1492 | 0.9465 |
78
+ | 0.3088 | 4.9446 | 22448 | 0.1487 | 0.9468 |
79
+ | 0.3047 | 5.1595 | 23424 | 0.1479 | 0.9468 |
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
 
81
 
82
  ### Framework versions
config.json CHANGED
@@ -17,10 +17,10 @@
17
  "global_attn_every_n_layers": 1,
18
  "global_rope_theta": 160000.0,
19
  "hidden_activation": "gelu",
20
- "hidden_size": 256,
21
  "initializer_cutoff_factor": 2.0,
22
  "initializer_range": 0.02,
23
- "intermediate_size": 384,
24
  "local_attention": 128,
25
  "local_rope_theta": 10000.0,
26
  "max_position_embeddings": 502,
@@ -29,8 +29,8 @@
29
  "model_type": "modernbert",
30
  "norm_bias": false,
31
  "norm_eps": 1e-05,
32
- "num_attention_heads": 4,
33
- "num_hidden_layers": 8,
34
  "pad_token_id": 1,
35
  "reference_compile": true,
36
  "repad_logits_with_grad": false,
 
17
  "global_attn_every_n_layers": 1,
18
  "global_rope_theta": 160000.0,
19
  "hidden_activation": "gelu",
20
+ "hidden_size": 768,
21
  "initializer_cutoff_factor": 2.0,
22
  "initializer_range": 0.02,
23
+ "intermediate_size": 1152,
24
  "local_attention": 128,
25
  "local_rope_theta": 10000.0,
26
  "max_position_embeddings": 502,
 
29
  "model_type": "modernbert",
30
  "norm_bias": false,
31
  "norm_eps": 1e-05,
32
+ "num_attention_heads": 12,
33
+ "num_hidden_layers": 22,
34
  "pad_token_id": 1,
35
  "reference_compile": true,
36
  "repad_logits_with_grad": false,
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e7026420b522f10b7f3e918f1714258a79fff6595dd39f8aa88b1097a1757e0a
3
- size 18195880
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9302ab93223b4470275b5e29b8f8a3f94180394328073b3244a9734dffddc7c
3
+ size 443955224
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ff593b8dc98307092bc953a0d2fad2da9c3ee4a23e38b37b7345e403d6365b62
3
  size 5905
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b16f71eeda3c291b5b689f85d3c9a2b70020ca8bfcb21c62875684cd68611c29
3
  size 5905