sparsetrace commited on
Commit
fdba41b
·
verified ·
1 Parent(s): 125a5e7

d32ft mirror latest (10 files)

Browse files
latest/meta_000250.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 250,
3
+ "val_bpb": 0.8097313191463176,
4
+ "model_config": {
5
+ "sequence_len": 2048,
6
+ "vocab_size": 65536,
7
+ "n_layer": 32,
8
+ "n_head": 16,
9
+ "n_kv_head": 16,
10
+ "n_embd": 2048,
11
+ "window_pattern": "L",
12
+ "attn_variant": "hmap",
13
+ "hmap_alpha": 1.0,
14
+ "hmap_beta": 1.0,
15
+ "witten": false,
16
+ "bidirectional": false
17
+ },
18
+ "user_config": {
19
+ "run": "dummy",
20
+ "device_type": "",
21
+ "fp8": false,
22
+ "fp8_recipe": "tensorwise",
23
+ "depth": 32,
24
+ "aspect_ratio": 64,
25
+ "head_dim": 128,
26
+ "max_seq_len": 2048,
27
+ "window_pattern": "L",
28
+ "attn_variant": "hmap",
29
+ "hmap_alpha": 1.0,
30
+ "hmap_beta": 1.0,
31
+ "witten": false,
32
+ "objective": "ar",
33
+ "mlm_mask_prob": 0.15,
34
+ "num_iterations": 1500,
35
+ "target_flops": -1.0,
36
+ "target_param_data_ratio": 12,
37
+ "device_batch_size": 2,
38
+ "total_batch_size": 524288,
39
+ "embedding_lr": 0.03,
40
+ "unembedding_lr": 0.0008,
41
+ "weight_decay": 0.0,
42
+ "matrix_lr": 0.002,
43
+ "scalar_lr": 0.05,
44
+ "warmup_steps": 40,
45
+ "warmdown_ratio": 0.0,
46
+ "final_lr_frac": 1.0,
47
+ "resume_from_step": -1,
48
+ "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt",
49
+ "eval_every": 100,
50
+ "eval_tokens": 41943040,
51
+ "core_metric_every": -1,
52
+ "core_metric_max_per_task": 500,
53
+ "sample_every": 250,
54
+ "save_every": 250,
55
+ "model_tag": "d32cmap"
56
+ },
57
+ "device_batch_size": 2,
58
+ "max_seq_len": 2048,
59
+ "total_batch_size": 524288,
60
+ "dataloader_state_dict": {
61
+ "pq_idx": 3,
62
+ "rg_idx": 8,
63
+ "epoch": 1
64
+ },
65
+ "loop_state": {
66
+ "min_val_bpb": 0.8097313191463176,
67
+ "smooth_train_loss": 2.700333229256745,
68
+ "total_training_time": 598.344233751297
69
+ }
70
+ }
latest/model_000250.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:95a7e29b2260b235cacaf2d6aedb36084fd4c8de7ec13f867f4d3d23e092d454
3
+ size 11542830898
latest/optim_000250_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe729274477222ffb8cddbd476581e0e3a20267edeabd74a83b4be2518c8333e
3
+ size 2080789141
latest/optim_000250_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f27bc97f8403bd4f5724d5247144209dfdb0a6466f00223978a9815f08f8fd1b
3
+ size 2080789141
latest/optim_000250_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:312e896aad5fdc65f7e5ad567f34844aaa19e897cd4a3f2db8e3ffed47e5bd4a
3
+ size 2080789141
latest/optim_000250_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5924eb05e777dd3778ded375867ca7ed1dc281fe8899b45ca9159ba819e18b5
3
+ size 2080789141
latest/optim_000250_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24a10e4da6e870f76bbf7b42416f27db9114b2372f10fa86d530cf5ff844578d
3
+ size 2080789141
latest/optim_000250_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56c35b00ed1ab556eb62da80ed24220a5f0e1fcb77454784cc9a3ae48e0447a3
3
+ size 2080789141
latest/optim_000250_rank6.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e29731451bc9260990ba9165970e3852b331fbf160c607da8a3f1c93a5298cf
3
+ size 2080789141
latest/optim_000250_rank7.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f1c97adcd2a784278a63e709a86b0c181b11cc116e5c0609066903ec67822ec
3
+ size 2080789141