fabikru commited on
Commit
c9caeb2
·
verified ·
1 Parent(s): 60c6195

fabikru/pubchem_and_chembl-2025-randomized-smiles-cleaned_modern_smiles_bert_4.5M_character-level_MLM0.3_new

Browse files
Files changed (4) hide show
  1. README.md +46 -42
  2. config.json +4 -4
  3. model.safetensors +2 -2
  4. training_args.bin +1 -1
README.md CHANGED
@@ -16,8 +16,8 @@ should probably proofread and complete it, then remove this comment. -->
16
 
17
  This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
- - Loss: 0.1419
20
- - Accuracy: 0.9500
21
 
22
  ## Model description
23
 
@@ -36,12 +36,12 @@ More information needed
36
  ### Training hyperparameters
37
 
38
  The following hyperparameters were used during training:
39
- - learning_rate: 0.013613
40
- - train_batch_size: 512
41
- - eval_batch_size: 512
42
  - seed: 42
43
- - gradient_accumulation_steps: 8
44
- - total_train_batch_size: 4096
45
  - optimizer: Use OptimizerNames.SCHEDULE_FREE_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
46
  - lr_scheduler_type: constant
47
  - lr_scheduler_warmup_steps: 1000
@@ -52,41 +52,45 @@ The following hyperparameters were used during training:
52
 
53
  | Training Loss | Epoch | Step | Validation Loss | Accuracy |
54
  |:-------------:|:------:|:----:|:---------------:|:--------:|
55
- | No log | 0 | 0 | 4.5077 | 0.0016 |
56
- | No log | 0.0044 | 122 | 0.7554 | 0.7645 |
57
- | No log | 0.0087 | 244 | 0.4784 | 0.8435 |
58
- | No log | 0.0131 | 366 | 0.3801 | 0.8737 |
59
- | No log | 0.0175 | 488 | 0.3245 | 0.8914 |
60
- | 6.1668 | 0.0218 | 610 | 0.2886 | 0.9028 |
61
- | 6.1668 | 0.0262 | 732 | 0.2692 | 0.9092 |
62
- | 6.1668 | 0.0306 | 854 | 0.2562 | 0.9136 |
63
- | 6.1668 | 0.0349 | 976 | 0.2327 | 0.9208 |
64
- | 2.441 | 0.0393 | 1098 | 0.2213 | 0.9247 |
65
- | 2.441 | 0.0437 | 1220 | 0.2112 | 0.9281 |
66
- | 2.441 | 0.0480 | 1342 | 0.2020 | 0.9308 |
67
- | 2.441 | 0.0524 | 1464 | 0.1933 | 0.9335 |
68
- | 1.7723 | 0.0568 | 1586 | 0.1874 | 0.9356 |
69
- | 1.7723 | 0.0611 | 1708 | 0.1809 | 0.9375 |
70
- | 1.7723 | 0.0655 | 1830 | 0.1782 | 0.9386 |
71
- | 1.7723 | 0.0699 | 1952 | 0.1749 | 0.9398 |
72
- | 1.5003 | 0.0743 | 2074 | 0.1712 | 0.9408 |
73
- | 1.5003 | 0.0786 | 2196 | 0.1689 | 0.9416 |
74
- | 1.5003 | 0.0830 | 2318 | 0.1656 | 0.9427 |
75
- | 1.5003 | 0.0874 | 2440 | 0.1640 | 0.9433 |
76
- | 1.3816 | 0.0917 | 2562 | 0.1620 | 0.9437 |
77
- | 1.3816 | 0.0961 | 2684 | 0.1590 | 0.9447 |
78
- | 1.3816 | 0.1005 | 2806 | 0.1568 | 0.9458 |
79
- | 1.3816 | 0.1048 | 2928 | 0.1547 | 0.9462 |
80
- | 1.3061 | 0.1092 | 3050 | 0.1535 | 0.9471 |
81
- | 1.3061 | 0.1136 | 3172 | 0.1524 | 0.9469 |
82
- | 1.3061 | 0.1179 | 3294 | 0.1497 | 0.9480 |
83
- | 1.3061 | 0.1223 | 3416 | 0.1482 | 0.9482 |
84
- | 1.2551 | 0.1267 | 3538 | 0.1471 | 0.9486 |
85
- | 1.2551 | 0.1310 | 3660 | 0.1461 | 0.9492 |
86
- | 1.2551 | 0.1354 | 3782 | 0.1450 | 0.9493 |
87
- | 1.2551 | 0.1398 | 3904 | 0.1438 | 0.9501 |
88
- | 1.2305 | 0.1441 | 4026 | 0.1430 | 0.9502 |
89
- | 1.2305 | 0.1485 | 4148 | 0.1421 | 0.9501 |
 
 
 
 
90
 
91
 
92
  ### Framework versions
 
16
 
17
  This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
+ - Loss: 0.1735
20
+ - Accuracy: 0.9399
21
 
22
  ## Model description
23
 
 
36
  ### Training hyperparameters
37
 
38
  The following hyperparameters were used during training:
39
+ - learning_rate: 0.032227
40
+ - train_batch_size: 1028
41
+ - eval_batch_size: 1028
42
  - seed: 42
43
+ - gradient_accumulation_steps: 4
44
+ - total_train_batch_size: 4112
45
  - optimizer: Use OptimizerNames.SCHEDULE_FREE_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
46
  - lr_scheduler_type: constant
47
  - lr_scheduler_warmup_steps: 1000
 
52
 
53
  | Training Loss | Epoch | Step | Validation Loss | Accuracy |
54
  |:-------------:|:------:|:----:|:---------------:|:--------:|
55
+ | No log | 0 | 0 | 4.4310 | 0.0041 |
56
+ | No log | 0.0043 | 121 | 0.7897 | 0.7545 |
57
+ | No log | 0.0087 | 242 | 0.5334 | 0.8269 |
58
+ | No log | 0.0130 | 363 | 0.4359 | 0.8565 |
59
+ | No log | 0.0174 | 484 | 0.3745 | 0.8758 |
60
+ | 3.2501 | 0.0217 | 605 | 0.3285 | 0.8900 |
61
+ | 3.2501 | 0.0261 | 726 | 0.2951 | 0.9012 |
62
+ | 3.2501 | 0.0304 | 847 | 0.2984 | 0.9001 |
63
+ | 3.2501 | 0.0348 | 968 | 0.2718 | 0.9084 |
64
+ | 1.4724 | 0.0391 | 1089 | 0.2632 | 0.9108 |
65
+ | 1.4724 | 0.0435 | 1210 | 0.2468 | 0.9164 |
66
+ | 1.4724 | 0.0478 | 1331 | 0.2449 | 0.9167 |
67
+ | 1.4724 | 0.0522 | 1452 | 0.2366 | 0.9196 |
68
+ | 1.0732 | 0.0565 | 1573 | 0.2255 | 0.9230 |
69
+ | 1.0732 | 0.0609 | 1694 | 0.2225 | 0.9241 |
70
+ | 1.0732 | 0.0652 | 1815 | 0.2176 | 0.9255 |
71
+ | 1.0732 | 0.0696 | 1936 | 0.2115 | 0.9276 |
72
+ | 0.9202 | 0.0739 | 2057 | 0.2075 | 0.9287 |
73
+ | 0.9202 | 0.0783 | 2178 | 0.2061 | 0.9293 |
74
+ | 0.9202 | 0.0826 | 2299 | 0.2024 | 0.9304 |
75
+ | 0.9202 | 0.0870 | 2420 | 0.2020 | 0.9311 |
76
+ | 0.8715 | 0.0913 | 2541 | 0.1986 | 0.9321 |
77
+ | 0.8715 | 0.0957 | 2662 | 0.1973 | 0.9324 |
78
+ | 0.8715 | 0.1000 | 2783 | 0.1959 | 0.9328 |
79
+ | 0.8715 | 0.1044 | 2904 | 0.1932 | 0.9337 |
80
+ | 0.8152 | 0.1087 | 3025 | 0.1880 | 0.9354 |
81
+ | 0.8152 | 0.1131 | 3146 | 0.1883 | 0.9353 |
82
+ | 0.8152 | 0.1174 | 3267 | 0.1856 | 0.9360 |
83
+ | 0.8152 | 0.1218 | 3388 | 0.1855 | 0.9362 |
84
+ | 0.7847 | 0.1261 | 3509 | 0.1837 | 0.9369 |
85
+ | 0.7847 | 0.1305 | 3630 | 0.1826 | 0.9373 |
86
+ | 0.7847 | 0.1348 | 3751 | 0.1813 | 0.9375 |
87
+ | 0.7847 | 0.1392 | 3872 | 0.1809 | 0.9373 |
88
+ | 0.7847 | 0.1435 | 3993 | 0.1790 | 0.9381 |
89
+ | 0.7671 | 0.1479 | 4114 | 0.1769 | 0.9387 |
90
+ | 0.7671 | 0.1522 | 4235 | 0.1778 | 0.9384 |
91
+ | 0.7671 | 0.1566 | 4356 | 0.1734 | 0.9399 |
92
+ | 0.7671 | 0.1609 | 4477 | 0.1740 | 0.9397 |
93
+ | 0.741 | 0.1653 | 4598 | 0.1730 | 0.9405 |
94
 
95
 
96
  ### Framework versions
config.json CHANGED
@@ -17,10 +17,10 @@
17
  "global_attn_every_n_layers": 1,
18
  "global_rope_theta": 160000.0,
19
  "hidden_activation": "gelu",
20
- "hidden_size": 384,
21
  "initializer_cutoff_factor": 2.0,
22
  "initializer_range": 0.02,
23
- "intermediate_size": 576,
24
  "local_attention": 128,
25
  "local_rope_theta": 10000.0,
26
  "max_position_embeddings": 502,
@@ -29,8 +29,8 @@
29
  "model_type": "modernbert",
30
  "norm_bias": false,
31
  "norm_eps": 1e-05,
32
- "num_attention_heads": 6,
33
- "num_hidden_layers": 12,
34
  "pad_token_id": 1,
35
  "reference_compile": true,
36
  "repad_logits_with_grad": false,
 
17
  "global_attn_every_n_layers": 1,
18
  "global_rope_theta": 160000.0,
19
  "hidden_activation": "gelu",
20
+ "hidden_size": 256,
21
  "initializer_cutoff_factor": 2.0,
22
  "initializer_range": 0.02,
23
+ "intermediate_size": 384,
24
  "local_attention": 128,
25
  "local_rope_theta": 10000.0,
26
  "max_position_embeddings": 502,
 
29
  "model_type": "modernbert",
30
  "norm_bias": false,
31
  "norm_eps": 1e-05,
32
+ "num_attention_heads": 4,
33
+ "num_hidden_layers": 8,
34
  "pad_token_id": 1,
35
  "reference_compile": true,
36
  "repad_logits_with_grad": false,
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:25299cee186846543ce89883f2d179b134beba2c358b667fc07164fcef93da2c
3
- size 60925776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7026420b522f10b7f3e918f1714258a79fff6595dd39f8aa88b1097a1757e0a
3
+ size 18195880
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:366498b850f960ef56f01707b48553334142ff7a428172f1fed105a3ad646d0e
3
  size 5905
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff593b8dc98307092bc953a0d2fad2da9c3ee4a23e38b37b7345e403d6365b62
3
  size 5905