checkpoint step 1000
Browse files
training-runs/latest/args.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"mode": "pretrain",
|
| 3 |
+
"dataset": "HuggingFaceFW/fineweb",
|
| 4 |
+
"dataset_config": "sample-10BT",
|
| 5 |
+
"split": "train",
|
| 6 |
+
"text_field": "text",
|
| 7 |
+
"model_size": "small",
|
| 8 |
+
"block_size": 512,
|
| 9 |
+
"batch_size": 4,
|
| 10 |
+
"grad_accum": 8,
|
| 11 |
+
"max_steps": 20000,
|
| 12 |
+
"lr": 0.0003,
|
| 13 |
+
"min_lr": 3e-05,
|
| 14 |
+
"warmup_steps": 100,
|
| 15 |
+
"weight_decay": 0.1,
|
| 16 |
+
"grad_clip": 1.0,
|
| 17 |
+
"dropout": 0.1,
|
| 18 |
+
"eval_interval": 1000,
|
| 19 |
+
"eval_batches": 20,
|
| 20 |
+
"save_interval": 1000,
|
| 21 |
+
"log_interval": 50,
|
| 22 |
+
"pcs_a": 0.8309193524478643,
|
| 23 |
+
"pcs_b": 0.0,
|
| 24 |
+
"out_dir": "runs/latest",
|
| 25 |
+
"hub_repo_id": "Daisuke675/scugnizz",
|
| 26 |
+
"push": true,
|
| 27 |
+
"push_every_save": true,
|
| 28 |
+
"seed": 42,
|
| 29 |
+
"dtype": "auto",
|
| 30 |
+
"no_resume": false,
|
| 31 |
+
"no_sample": true,
|
| 32 |
+
"prompt": "User: What is artificial intelligence?\nAssistant:",
|
| 33 |
+
"max_new_tokens": 120
|
| 34 |
+
}
|
training-runs/latest/checkpoint_best.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7144561336f9849a7baa57e89918e29ad237214bca00af2dd7487f9ba8b588c5
|
| 3 |
+
size 614210639
|
training-runs/latest/checkpoint_last.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:873a32d60a9798e40b6b0e7f911da9202a9b14613d763ccc4ca7fd98b3d42712
|
| 3 |
+
size 614210639
|