Daisuke675 commited on
Commit
c460224
·
verified ·
1 Parent(s): b5535b6

checkpoint step 1000

Browse files
training-runs/latest/args.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mode": "pretrain",
3
+ "dataset": "HuggingFaceFW/fineweb",
4
+ "dataset_config": "sample-10BT",
5
+ "split": "train",
6
+ "text_field": "text",
7
+ "model_size": "small",
8
+ "block_size": 512,
9
+ "batch_size": 4,
10
+ "grad_accum": 8,
11
+ "max_steps": 20000,
12
+ "lr": 0.0003,
13
+ "min_lr": 3e-05,
14
+ "warmup_steps": 100,
15
+ "weight_decay": 0.1,
16
+ "grad_clip": 1.0,
17
+ "dropout": 0.1,
18
+ "eval_interval": 1000,
19
+ "eval_batches": 20,
20
+ "save_interval": 1000,
21
+ "log_interval": 50,
22
+ "pcs_a": 0.8309193524478643,
23
+ "pcs_b": 0.0,
24
+ "out_dir": "runs/latest",
25
+ "hub_repo_id": "Daisuke675/scugnizz",
26
+ "push": true,
27
+ "push_every_save": true,
28
+ "seed": 42,
29
+ "dtype": "auto",
30
+ "no_resume": false,
31
+ "no_sample": true,
32
+ "prompt": "User: What is artificial intelligence?\nAssistant:",
33
+ "max_new_tokens": 120
34
+ }
training-runs/latest/checkpoint_best.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7144561336f9849a7baa57e89918e29ad237214bca00af2dd7487f9ba8b588c5
3
+ size 614210639
training-runs/latest/checkpoint_last.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:873a32d60a9798e40b6b0e7f911da9202a9b14613d763ccc4ca7fd98b3d42712
3
+ size 614210639