Instructions to use Synth-Ecosystem/synthos with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Synth-Ecosystem/synthos with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("Synth-Ecosystem/synthos", device_map="auto") - Notebooks
- Google Colab
- Kaggle
tzar z commited on
Sovereign Genesis - checkpoint-6500 - Weights Only Search
Browse files- optimizer.pt +1 -1
- rng_state.pth +1 -1
- scheduler.pt +1 -1
- trainer_state.json +503 -3
optimizer.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 156887749
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47f67f06db84fcc8da54e972cc7699113d30b9ef90bbf64e4b4fbea0a13aa7eb
|
| 3 |
size 156887749
|
rng_state.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 14645
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:db33c6d7e1e2e9fca6e1172ccd6d0688542d2afe44ce414b1ef64b5b605800b3
|
| 3 |
size 14645
|
scheduler.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 1465
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:84132c3c9af421dcfdaff63d87654e6fbb1775927b228ae6f7089eb34cd9e03d
|
| 3 |
size 1465
|
trainer_state.json
CHANGED
|
@@ -2,9 +2,9 @@
|
|
| 2 |
"best_global_step": null,
|
| 3 |
"best_metric": null,
|
| 4 |
"best_model_checkpoint": null,
|
| 5 |
-
"epoch":
|
| 6 |
"eval_steps": 500,
|
| 7 |
-
"global_step":
|
| 8 |
"is_hyper_param_search": false,
|
| 9 |
"is_local_process_zero": true,
|
| 10 |
"is_world_process_zero": true,
|
|
@@ -6008,6 +6008,506 @@
|
|
| 6008 |
"mean_token_accuracy": 0.9227724611759186,
|
| 6009 |
"num_tokens": 83767681.0,
|
| 6010 |
"step": 6000
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6011 |
}
|
| 6012 |
],
|
| 6013 |
"logging_steps": 10,
|
|
@@ -6027,7 +6527,7 @@
|
|
| 6027 |
"attributes": {}
|
| 6028 |
}
|
| 6029 |
},
|
| 6030 |
-
"total_flos": 1.
|
| 6031 |
"train_batch_size": 4,
|
| 6032 |
"trial_name": null,
|
| 6033 |
"trial_params": null
|
|
|
|
| 2 |
"best_global_step": null,
|
| 3 |
"best_metric": null,
|
| 4 |
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 3.070382616910723,
|
| 6 |
"eval_steps": 500,
|
| 7 |
+
"global_step": 6500,
|
| 8 |
"is_hyper_param_search": false,
|
| 9 |
"is_local_process_zero": true,
|
| 10 |
"is_world_process_zero": true,
|
|
|
|
| 6008 |
"mean_token_accuracy": 0.9227724611759186,
|
| 6009 |
"num_tokens": 83767681.0,
|
| 6010 |
"step": 6000
|
| 6011 |
+
},
|
| 6012 |
+
{
|
| 6013 |
+
"entropy": 0.33384765721857546,
|
| 6014 |
+
"epoch": 2.838923004251299,
|
| 6015 |
+
"grad_norm": 0.0700036883354187,
|
| 6016 |
+
"learning_rate": 0.0001632206037732489,
|
| 6017 |
+
"loss": 0.2932481288909912,
|
| 6018 |
+
"mean_token_accuracy": 0.924432634562254,
|
| 6019 |
+
"num_tokens": 83909784.0,
|
| 6020 |
+
"step": 6010
|
| 6021 |
+
},
|
| 6022 |
+
{
|
| 6023 |
+
"entropy": 0.3035454021766782,
|
| 6024 |
+
"epoch": 2.843646669815777,
|
| 6025 |
+
"grad_norm": 0.059564728289842606,
|
| 6026 |
+
"learning_rate": 0.0001631052825962278,
|
| 6027 |
+
"loss": 0.276937460899353,
|
| 6028 |
+
"mean_token_accuracy": 0.9300597168505191,
|
| 6029 |
+
"num_tokens": 84050579.0,
|
| 6030 |
+
"step": 6020
|
| 6031 |
+
},
|
| 6032 |
+
{
|
| 6033 |
+
"entropy": 0.3417934877797961,
|
| 6034 |
+
"epoch": 2.848370335380255,
|
| 6035 |
+
"grad_norm": 0.08422790467739105,
|
| 6036 |
+
"learning_rate": 0.00016298982178958755,
|
| 6037 |
+
"loss": 0.30077276229858396,
|
| 6038 |
+
"mean_token_accuracy": 0.9234837457537651,
|
| 6039 |
+
"num_tokens": 84182932.0,
|
| 6040 |
+
"step": 6030
|
| 6041 |
+
},
|
| 6042 |
+
{
|
| 6043 |
+
"entropy": 0.3213076775893569,
|
| 6044 |
+
"epoch": 2.853094000944733,
|
| 6045 |
+
"grad_norm": 0.07368471473455429,
|
| 6046 |
+
"learning_rate": 0.00016287422160880195,
|
| 6047 |
+
"loss": 0.28111531734466555,
|
| 6048 |
+
"mean_token_accuracy": 0.9266740024089813,
|
| 6049 |
+
"num_tokens": 84324453.0,
|
| 6050 |
+
"step": 6040
|
| 6051 |
+
},
|
| 6052 |
+
{
|
| 6053 |
+
"entropy": 0.3168294267728925,
|
| 6054 |
+
"epoch": 2.857817666509211,
|
| 6055 |
+
"grad_norm": 0.07869284600019455,
|
| 6056 |
+
"learning_rate": 0.00016275848230965325,
|
| 6057 |
+
"loss": 0.2750498056411743,
|
| 6058 |
+
"mean_token_accuracy": 0.9285811208188534,
|
| 6059 |
+
"num_tokens": 84463560.0,
|
| 6060 |
+
"step": 6050
|
| 6061 |
+
},
|
| 6062 |
+
{
|
| 6063 |
+
"entropy": 0.3199483294971287,
|
| 6064 |
+
"epoch": 2.862541332073689,
|
| 6065 |
+
"grad_norm": 0.07346589118242264,
|
| 6066 |
+
"learning_rate": 0.00016264260414823145,
|
| 6067 |
+
"loss": 0.27833597660064696,
|
| 6068 |
+
"mean_token_accuracy": 0.9264893747866154,
|
| 6069 |
+
"num_tokens": 84604085.0,
|
| 6070 |
+
"step": 6060
|
| 6071 |
+
},
|
| 6072 |
+
{
|
| 6073 |
+
"entropy": 0.31892386665567757,
|
| 6074 |
+
"epoch": 2.8672649976381672,
|
| 6075 |
+
"grad_norm": 0.0658896416425705,
|
| 6076 |
+
"learning_rate": 0.0001625265873809339,
|
| 6077 |
+
"loss": 0.272175931930542,
|
| 6078 |
+
"mean_token_accuracy": 0.9286142311990261,
|
| 6079 |
+
"num_tokens": 84737551.0,
|
| 6080 |
+
"step": 6070
|
| 6081 |
+
},
|
| 6082 |
+
{
|
| 6083 |
+
"entropy": 0.3409381164237857,
|
| 6084 |
+
"epoch": 2.8719886632026452,
|
| 6085 |
+
"grad_norm": 0.05578494071960449,
|
| 6086 |
+
"learning_rate": 0.0001624104322644645,
|
| 6087 |
+
"loss": 0.3090266227722168,
|
| 6088 |
+
"mean_token_accuracy": 0.9216448545455933,
|
| 6089 |
+
"num_tokens": 84875724.0,
|
| 6090 |
+
"step": 6080
|
| 6091 |
+
},
|
| 6092 |
+
{
|
| 6093 |
+
"entropy": 0.3283074229024351,
|
| 6094 |
+
"epoch": 2.8767123287671232,
|
| 6095 |
+
"grad_norm": 0.09109394997358322,
|
| 6096 |
+
"learning_rate": 0.00016229413905583342,
|
| 6097 |
+
"loss": 0.2874417304992676,
|
| 6098 |
+
"mean_token_accuracy": 0.9258060708642006,
|
| 6099 |
+
"num_tokens": 85013623.0,
|
| 6100 |
+
"step": 6090
|
| 6101 |
+
},
|
| 6102 |
+
{
|
| 6103 |
+
"entropy": 0.2992855066433549,
|
| 6104 |
+
"epoch": 2.8814359943316012,
|
| 6105 |
+
"grad_norm": 0.0764261856675148,
|
| 6106 |
+
"learning_rate": 0.00016217770801235624,
|
| 6107 |
+
"loss": 0.2696357727050781,
|
| 6108 |
+
"mean_token_accuracy": 0.929871492832899,
|
| 6109 |
+
"num_tokens": 85147931.0,
|
| 6110 |
+
"step": 6100
|
| 6111 |
+
},
|
| 6112 |
+
{
|
| 6113 |
+
"entropy": 0.29618923347443343,
|
| 6114 |
+
"epoch": 2.8861596598960793,
|
| 6115 |
+
"grad_norm": 0.06750992685556412,
|
| 6116 |
+
"learning_rate": 0.00016206113939165365,
|
| 6117 |
+
"loss": 0.2536092519760132,
|
| 6118 |
+
"mean_token_accuracy": 0.9344585418701172,
|
| 6119 |
+
"num_tokens": 85284684.0,
|
| 6120 |
+
"step": 6110
|
| 6121 |
+
},
|
| 6122 |
+
{
|
| 6123 |
+
"entropy": 0.2895301982760429,
|
| 6124 |
+
"epoch": 2.8908833254605573,
|
| 6125 |
+
"grad_norm": 0.07175379991531372,
|
| 6126 |
+
"learning_rate": 0.00016194443345165062,
|
| 6127 |
+
"loss": 0.25012803077697754,
|
| 6128 |
+
"mean_token_accuracy": 0.9349378451704979,
|
| 6129 |
+
"num_tokens": 85425297.0,
|
| 6130 |
+
"step": 6120
|
| 6131 |
+
},
|
| 6132 |
+
{
|
| 6133 |
+
"entropy": 0.28265368901193144,
|
| 6134 |
+
"epoch": 2.8956069910250353,
|
| 6135 |
+
"grad_norm": 0.06473183631896973,
|
| 6136 |
+
"learning_rate": 0.00016182759045057607,
|
| 6137 |
+
"loss": 0.2529444217681885,
|
| 6138 |
+
"mean_token_accuracy": 0.9354198552668095,
|
| 6139 |
+
"num_tokens": 85567865.0,
|
| 6140 |
+
"step": 6130
|
| 6141 |
+
},
|
| 6142 |
+
{
|
| 6143 |
+
"entropy": 0.3008987005800009,
|
| 6144 |
+
"epoch": 2.9003306565895137,
|
| 6145 |
+
"grad_norm": 0.07270916551351547,
|
| 6146 |
+
"learning_rate": 0.00016171061064696208,
|
| 6147 |
+
"loss": 0.2582638025283813,
|
| 6148 |
+
"mean_token_accuracy": 0.9329213611781597,
|
| 6149 |
+
"num_tokens": 85710103.0,
|
| 6150 |
+
"step": 6140
|
| 6151 |
+
},
|
| 6152 |
+
{
|
| 6153 |
+
"entropy": 0.3001433851197362,
|
| 6154 |
+
"epoch": 2.9050543221539913,
|
| 6155 |
+
"grad_norm": 0.0838063433766365,
|
| 6156 |
+
"learning_rate": 0.00016159349429964358,
|
| 6157 |
+
"loss": 0.2681830644607544,
|
| 6158 |
+
"mean_token_accuracy": 0.933569273352623,
|
| 6159 |
+
"num_tokens": 85852779.0,
|
| 6160 |
+
"step": 6150
|
| 6161 |
+
},
|
| 6162 |
+
{
|
| 6163 |
+
"entropy": 0.3165293637663126,
|
| 6164 |
+
"epoch": 2.9097779877184697,
|
| 6165 |
+
"grad_norm": 0.06948206573724747,
|
| 6166 |
+
"learning_rate": 0.00016147624166775744,
|
| 6167 |
+
"loss": 0.2737043619155884,
|
| 6168 |
+
"mean_token_accuracy": 0.9284098230302333,
|
| 6169 |
+
"num_tokens": 85994124.0,
|
| 6170 |
+
"step": 6160
|
| 6171 |
+
},
|
| 6172 |
+
{
|
| 6173 |
+
"entropy": 0.32346988916397096,
|
| 6174 |
+
"epoch": 2.9145016532829473,
|
| 6175 |
+
"grad_norm": 0.08191309869289398,
|
| 6176 |
+
"learning_rate": 0.0001613588530107422,
|
| 6177 |
+
"loss": 0.275394868850708,
|
| 6178 |
+
"mean_token_accuracy": 0.9277596913278103,
|
| 6179 |
+
"num_tokens": 86130678.0,
|
| 6180 |
+
"step": 6170
|
| 6181 |
+
},
|
| 6182 |
+
{
|
| 6183 |
+
"entropy": 0.288114645332098,
|
| 6184 |
+
"epoch": 2.9192253188474258,
|
| 6185 |
+
"grad_norm": 0.06179473176598549,
|
| 6186 |
+
"learning_rate": 0.00016124132858833735,
|
| 6187 |
+
"loss": 0.25613381862640383,
|
| 6188 |
+
"mean_token_accuracy": 0.9330091632902622,
|
| 6189 |
+
"num_tokens": 86264282.0,
|
| 6190 |
+
"step": 6180
|
| 6191 |
+
},
|
| 6192 |
+
{
|
| 6193 |
+
"entropy": 0.3138307509943843,
|
| 6194 |
+
"epoch": 2.9239489844119038,
|
| 6195 |
+
"grad_norm": 0.08615489304065704,
|
| 6196 |
+
"learning_rate": 0.00016112366866058282,
|
| 6197 |
+
"loss": 0.27022275924682615,
|
| 6198 |
+
"mean_token_accuracy": 0.9309975825250149,
|
| 6199 |
+
"num_tokens": 86395752.0,
|
| 6200 |
+
"step": 6190
|
| 6201 |
+
},
|
| 6202 |
+
{
|
| 6203 |
+
"entropy": 0.3056884821504354,
|
| 6204 |
+
"epoch": 2.9286726499763818,
|
| 6205 |
+
"grad_norm": 0.054475221782922745,
|
| 6206 |
+
"learning_rate": 0.0001610058734878182,
|
| 6207 |
+
"loss": 0.2727755308151245,
|
| 6208 |
+
"mean_token_accuracy": 0.9301555290818214,
|
| 6209 |
+
"num_tokens": 86535633.0,
|
| 6210 |
+
"step": 6200
|
| 6211 |
+
},
|
| 6212 |
+
{
|
| 6213 |
+
"entropy": 0.28974386136978864,
|
| 6214 |
+
"epoch": 2.9333963155408598,
|
| 6215 |
+
"grad_norm": 0.07793719321489334,
|
| 6216 |
+
"learning_rate": 0.00016088794333068252,
|
| 6217 |
+
"loss": 0.2515974521636963,
|
| 6218 |
+
"mean_token_accuracy": 0.9339422516524791,
|
| 6219 |
+
"num_tokens": 86679523.0,
|
| 6220 |
+
"step": 6210
|
| 6221 |
+
},
|
| 6222 |
+
{
|
| 6223 |
+
"entropy": 0.3044612675905228,
|
| 6224 |
+
"epoch": 2.938119981105338,
|
| 6225 |
+
"grad_norm": 0.07805371284484863,
|
| 6226 |
+
"learning_rate": 0.00016076987845011344,
|
| 6227 |
+
"loss": 0.26934287548065183,
|
| 6228 |
+
"mean_token_accuracy": 0.9311383672058582,
|
| 6229 |
+
"num_tokens": 86818047.0,
|
| 6230 |
+
"step": 6220
|
| 6231 |
+
},
|
| 6232 |
+
{
|
| 6233 |
+
"entropy": 0.34343143105506896,
|
| 6234 |
+
"epoch": 2.942843646669816,
|
| 6235 |
+
"grad_norm": 0.07105180621147156,
|
| 6236 |
+
"learning_rate": 0.00016065167910734666,
|
| 6237 |
+
"loss": 0.3158273458480835,
|
| 6238 |
+
"mean_token_accuracy": 0.9232825860381126,
|
| 6239 |
+
"num_tokens": 86960750.0,
|
| 6240 |
+
"step": 6230
|
| 6241 |
+
},
|
| 6242 |
+
{
|
| 6243 |
+
"entropy": 0.3021335743367672,
|
| 6244 |
+
"epoch": 2.947567312234294,
|
| 6245 |
+
"grad_norm": 0.07468222081661224,
|
| 6246 |
+
"learning_rate": 0.00016053334556391535,
|
| 6247 |
+
"loss": 0.25798649787902833,
|
| 6248 |
+
"mean_token_accuracy": 0.9327261500060559,
|
| 6249 |
+
"num_tokens": 87101702.0,
|
| 6250 |
+
"step": 6240
|
| 6251 |
+
},
|
| 6252 |
+
{
|
| 6253 |
+
"entropy": 0.2945020263083279,
|
| 6254 |
+
"epoch": 2.952290977798772,
|
| 6255 |
+
"grad_norm": 0.05098683387041092,
|
| 6256 |
+
"learning_rate": 0.00016041487808164982,
|
| 6257 |
+
"loss": 0.2529811143875122,
|
| 6258 |
+
"mean_token_accuracy": 0.9338640421628952,
|
| 6259 |
+
"num_tokens": 87237214.0,
|
| 6260 |
+
"step": 6250
|
| 6261 |
+
},
|
| 6262 |
+
{
|
| 6263 |
+
"entropy": 0.30731857661157846,
|
| 6264 |
+
"epoch": 2.95701464336325,
|
| 6265 |
+
"grad_norm": 0.07433168590068817,
|
| 6266 |
+
"learning_rate": 0.0001602962769226765,
|
| 6267 |
+
"loss": 0.2620837926864624,
|
| 6268 |
+
"mean_token_accuracy": 0.9326300747692585,
|
| 6269 |
+
"num_tokens": 87378353.0,
|
| 6270 |
+
"step": 6260
|
| 6271 |
+
},
|
| 6272 |
+
{
|
| 6273 |
+
"entropy": 0.3260970426723361,
|
| 6274 |
+
"epoch": 2.961738308927728,
|
| 6275 |
+
"grad_norm": 0.06732673197984695,
|
| 6276 |
+
"learning_rate": 0.0001601775423494178,
|
| 6277 |
+
"loss": 0.292596435546875,
|
| 6278 |
+
"mean_token_accuracy": 0.9259773470461369,
|
| 6279 |
+
"num_tokens": 87517051.0,
|
| 6280 |
+
"step": 6270
|
| 6281 |
+
},
|
| 6282 |
+
{
|
| 6283 |
+
"entropy": 0.31896489188075067,
|
| 6284 |
+
"epoch": 2.966461974492206,
|
| 6285 |
+
"grad_norm": 0.09348985552787781,
|
| 6286 |
+
"learning_rate": 0.00016005867462459117,
|
| 6287 |
+
"loss": 0.28274738788604736,
|
| 6288 |
+
"mean_token_accuracy": 0.9286909833550453,
|
| 6289 |
+
"num_tokens": 87653424.0,
|
| 6290 |
+
"step": 6280
|
| 6291 |
+
},
|
| 6292 |
+
{
|
| 6293 |
+
"entropy": 0.32120455633848904,
|
| 6294 |
+
"epoch": 2.971185640056684,
|
| 6295 |
+
"grad_norm": 0.07910658419132233,
|
| 6296 |
+
"learning_rate": 0.00015993967401120877,
|
| 6297 |
+
"loss": 0.27887530326843263,
|
| 6298 |
+
"mean_token_accuracy": 0.9292895056307315,
|
| 6299 |
+
"num_tokens": 87792261.0,
|
| 6300 |
+
"step": 6290
|
| 6301 |
+
},
|
| 6302 |
+
{
|
| 6303 |
+
"entropy": 0.3166860887780786,
|
| 6304 |
+
"epoch": 2.975909305621162,
|
| 6305 |
+
"grad_norm": 0.07976067811250687,
|
| 6306 |
+
"learning_rate": 0.00015982054077257678,
|
| 6307 |
+
"loss": 0.2847684621810913,
|
| 6308 |
+
"mean_token_accuracy": 0.9279345542192459,
|
| 6309 |
+
"num_tokens": 87924254.0,
|
| 6310 |
+
"step": 6300
|
| 6311 |
+
},
|
| 6312 |
+
{
|
| 6313 |
+
"entropy": 0.30990776466205716,
|
| 6314 |
+
"epoch": 2.9806329711856403,
|
| 6315 |
+
"grad_norm": 0.06893040984869003,
|
| 6316 |
+
"learning_rate": 0.00015970127517229483,
|
| 6317 |
+
"loss": 0.2684711217880249,
|
| 6318 |
+
"mean_token_accuracy": 0.930588485300541,
|
| 6319 |
+
"num_tokens": 88064452.0,
|
| 6320 |
+
"step": 6310
|
| 6321 |
+
},
|
| 6322 |
+
{
|
| 6323 |
+
"entropy": 0.29596504513174293,
|
| 6324 |
+
"epoch": 2.985356636750118,
|
| 6325 |
+
"grad_norm": 0.08254679292440414,
|
| 6326 |
+
"learning_rate": 0.00015958187747425537,
|
| 6327 |
+
"loss": 0.2469400405883789,
|
| 6328 |
+
"mean_token_accuracy": 0.9356708757579326,
|
| 6329 |
+
"num_tokens": 88197423.0,
|
| 6330 |
+
"step": 6320
|
| 6331 |
+
},
|
| 6332 |
+
{
|
| 6333 |
+
"entropy": 0.3303051806986332,
|
| 6334 |
+
"epoch": 2.9900803023145963,
|
| 6335 |
+
"grad_norm": 0.05904023349285126,
|
| 6336 |
+
"learning_rate": 0.00015946234794264323,
|
| 6337 |
+
"loss": 0.2958174705505371,
|
| 6338 |
+
"mean_token_accuracy": 0.926980547606945,
|
| 6339 |
+
"num_tokens": 88340784.0,
|
| 6340 |
+
"step": 6330
|
| 6341 |
+
},
|
| 6342 |
+
{
|
| 6343 |
+
"entropy": 0.31454561464488506,
|
| 6344 |
+
"epoch": 2.9948039678790743,
|
| 6345 |
+
"grad_norm": 0.07458409667015076,
|
| 6346 |
+
"learning_rate": 0.00015934268684193484,
|
| 6347 |
+
"loss": 0.27035863399505616,
|
| 6348 |
+
"mean_token_accuracy": 0.9297575414180755,
|
| 6349 |
+
"num_tokens": 88480345.0,
|
| 6350 |
+
"step": 6340
|
| 6351 |
+
},
|
| 6352 |
+
{
|
| 6353 |
+
"entropy": 0.29515541810542345,
|
| 6354 |
+
"epoch": 2.9995276334435523,
|
| 6355 |
+
"grad_norm": 0.07899592071771622,
|
| 6356 |
+
"learning_rate": 0.00015922289443689785,
|
| 6357 |
+
"loss": 0.25933036804199217,
|
| 6358 |
+
"mean_token_accuracy": 0.932505565136671,
|
| 6359 |
+
"num_tokens": 88619762.0,
|
| 6360 |
+
"step": 6350
|
| 6361 |
+
},
|
| 6362 |
+
{
|
| 6363 |
+
"entropy": 0.29944039676338435,
|
| 6364 |
+
"epoch": 3.0042512990080303,
|
| 6365 |
+
"grad_norm": 0.07667111605405807,
|
| 6366 |
+
"learning_rate": 0.00015910297099259034,
|
| 6367 |
+
"loss": 0.2365949869155884,
|
| 6368 |
+
"mean_token_accuracy": 0.9397827386856079,
|
| 6369 |
+
"num_tokens": 88754353.0,
|
| 6370 |
+
"step": 6360
|
| 6371 |
+
},
|
| 6372 |
+
{
|
| 6373 |
+
"entropy": 0.27041827561333776,
|
| 6374 |
+
"epoch": 3.0089749645725083,
|
| 6375 |
+
"grad_norm": 0.09656394273042679,
|
| 6376 |
+
"learning_rate": 0.00015898291677436042,
|
| 6377 |
+
"loss": 0.22616770267486572,
|
| 6378 |
+
"mean_token_accuracy": 0.9424898579716683,
|
| 6379 |
+
"num_tokens": 88890997.0,
|
| 6380 |
+
"step": 6370
|
| 6381 |
+
},
|
| 6382 |
+
{
|
| 6383 |
+
"entropy": 0.28427893491461875,
|
| 6384 |
+
"epoch": 3.0136986301369864,
|
| 6385 |
+
"grad_norm": 0.06355519592761993,
|
| 6386 |
+
"learning_rate": 0.00015886273204784548,
|
| 6387 |
+
"loss": 0.2329385757446289,
|
| 6388 |
+
"mean_token_accuracy": 0.9410013981163502,
|
| 6389 |
+
"num_tokens": 89032170.0,
|
| 6390 |
+
"step": 6380
|
| 6391 |
+
},
|
| 6392 |
+
{
|
| 6393 |
+
"entropy": 0.2778877759352326,
|
| 6394 |
+
"epoch": 3.0184222957014644,
|
| 6395 |
+
"grad_norm": 0.07877601683139801,
|
| 6396 |
+
"learning_rate": 0.0001587424170789717,
|
| 6397 |
+
"loss": 0.2319040298461914,
|
| 6398 |
+
"mean_token_accuracy": 0.9415215879678727,
|
| 6399 |
+
"num_tokens": 89178899.0,
|
| 6400 |
+
"step": 6390
|
| 6401 |
+
},
|
| 6402 |
+
{
|
| 6403 |
+
"entropy": 0.2784922879189253,
|
| 6404 |
+
"epoch": 3.0231459612659424,
|
| 6405 |
+
"grad_norm": 0.06589914858341217,
|
| 6406 |
+
"learning_rate": 0.0001586219721339535,
|
| 6407 |
+
"loss": 0.2364208459854126,
|
| 6408 |
+
"mean_token_accuracy": 0.9419798776507378,
|
| 6409 |
+
"num_tokens": 89325229.0,
|
| 6410 |
+
"step": 6400
|
| 6411 |
+
},
|
| 6412 |
+
{
|
| 6413 |
+
"entropy": 0.29138854378834367,
|
| 6414 |
+
"epoch": 3.0278696268304204,
|
| 6415 |
+
"grad_norm": 0.06874479353427887,
|
| 6416 |
+
"learning_rate": 0.00015850139747929283,
|
| 6417 |
+
"loss": 0.22848091125488282,
|
| 6418 |
+
"mean_token_accuracy": 0.9409735448658466,
|
| 6419 |
+
"num_tokens": 89467865.0,
|
| 6420 |
+
"step": 6410
|
| 6421 |
+
},
|
| 6422 |
+
{
|
| 6423 |
+
"entropy": 0.2573707280680537,
|
| 6424 |
+
"epoch": 3.0325932923948984,
|
| 6425 |
+
"grad_norm": 0.05927377566695213,
|
| 6426 |
+
"learning_rate": 0.00015838069338177863,
|
| 6427 |
+
"loss": 0.20626981258392335,
|
| 6428 |
+
"mean_token_accuracy": 0.9469245828688144,
|
| 6429 |
+
"num_tokens": 89601896.0,
|
| 6430 |
+
"step": 6420
|
| 6431 |
+
},
|
| 6432 |
+
{
|
| 6433 |
+
"entropy": 0.24344939803704618,
|
| 6434 |
+
"epoch": 3.0373169579593764,
|
| 6435 |
+
"grad_norm": 0.06410811841487885,
|
| 6436 |
+
"learning_rate": 0.00015825986010848634,
|
| 6437 |
+
"loss": 0.19441936016082764,
|
| 6438 |
+
"mean_token_accuracy": 0.9505223080515861,
|
| 6439 |
+
"num_tokens": 89742477.0,
|
| 6440 |
+
"step": 6430
|
| 6441 |
+
},
|
| 6442 |
+
{
|
| 6443 |
+
"entropy": 0.25771507173776625,
|
| 6444 |
+
"epoch": 3.0420406235238544,
|
| 6445 |
+
"grad_norm": 0.053573086857795715,
|
| 6446 |
+
"learning_rate": 0.00015813889792677706,
|
| 6447 |
+
"loss": 0.21072278022766114,
|
| 6448 |
+
"mean_token_accuracy": 0.9468448996543884,
|
| 6449 |
+
"num_tokens": 89872459.0,
|
| 6450 |
+
"step": 6440
|
| 6451 |
+
},
|
| 6452 |
+
{
|
| 6453 |
+
"entropy": 0.24889703365042806,
|
| 6454 |
+
"epoch": 3.0467642890883324,
|
| 6455 |
+
"grad_norm": 0.07980092614889145,
|
| 6456 |
+
"learning_rate": 0.0001580178071042973,
|
| 6457 |
+
"loss": 0.2015373945236206,
|
| 6458 |
+
"mean_token_accuracy": 0.9471607655286789,
|
| 6459 |
+
"num_tokens": 90011569.0,
|
| 6460 |
+
"step": 6450
|
| 6461 |
+
},
|
| 6462 |
+
{
|
| 6463 |
+
"entropy": 0.2616048602387309,
|
| 6464 |
+
"epoch": 3.0514879546528104,
|
| 6465 |
+
"grad_norm": 0.06496746838092804,
|
| 6466 |
+
"learning_rate": 0.00015789658790897807,
|
| 6467 |
+
"loss": 0.21244680881500244,
|
| 6468 |
+
"mean_token_accuracy": 0.9469091944396496,
|
| 6469 |
+
"num_tokens": 90150606.0,
|
| 6470 |
+
"step": 6460
|
| 6471 |
+
},
|
| 6472 |
+
{
|
| 6473 |
+
"entropy": 0.266824578307569,
|
| 6474 |
+
"epoch": 3.0562116202172884,
|
| 6475 |
+
"grad_norm": 0.06718926131725311,
|
| 6476 |
+
"learning_rate": 0.0001577752406090345,
|
| 6477 |
+
"loss": 0.22119715213775634,
|
| 6478 |
+
"mean_token_accuracy": 0.9423008799552918,
|
| 6479 |
+
"num_tokens": 90276269.0,
|
| 6480 |
+
"step": 6470
|
| 6481 |
+
},
|
| 6482 |
+
{
|
| 6483 |
+
"entropy": 0.27743001822382213,
|
| 6484 |
+
"epoch": 3.0609352857817664,
|
| 6485 |
+
"grad_norm": 0.07867974787950516,
|
| 6486 |
+
"learning_rate": 0.0001576537654729652,
|
| 6487 |
+
"loss": 0.2343437910079956,
|
| 6488 |
+
"mean_token_accuracy": 0.9408437065780163,
|
| 6489 |
+
"num_tokens": 90412699.0,
|
| 6490 |
+
"step": 6480
|
| 6491 |
+
},
|
| 6492 |
+
{
|
| 6493 |
+
"entropy": 0.26375128515064716,
|
| 6494 |
+
"epoch": 3.065658951346245,
|
| 6495 |
+
"grad_norm": 0.05987164005637169,
|
| 6496 |
+
"learning_rate": 0.0001575321627695515,
|
| 6497 |
+
"loss": 0.2072808027267456,
|
| 6498 |
+
"mean_token_accuracy": 0.9460294567048549,
|
| 6499 |
+
"num_tokens": 90547399.0,
|
| 6500 |
+
"step": 6490
|
| 6501 |
+
},
|
| 6502 |
+
{
|
| 6503 |
+
"entropy": 0.2649696357548237,
|
| 6504 |
+
"epoch": 3.070382616910723,
|
| 6505 |
+
"grad_norm": 0.06754233688116074,
|
| 6506 |
+
"learning_rate": 0.00015741043276785709,
|
| 6507 |
+
"loss": 0.21425178050994872,
|
| 6508 |
+
"mean_token_accuracy": 0.9447133392095566,
|
| 6509 |
+
"num_tokens": 90687331.0,
|
| 6510 |
+
"step": 6500
|
| 6511 |
}
|
| 6512 |
],
|
| 6513 |
"logging_steps": 10,
|
|
|
|
| 6527 |
"attributes": {}
|
| 6528 |
}
|
| 6529 |
},
|
| 6530 |
+
"total_flos": 1.3672583194431283e+18,
|
| 6531 |
"train_batch_size": 4,
|
| 6532 |
"trial_name": null,
|
| 6533 |
"trial_params": null
|