Compactbot commited on
Commit
d2df945
·
verified ·
1 Parent(s): 364aabf

Update config to v7 checkpoint (step 30000, best_val 3.6892)

Browse files
Files changed (1) hide show
  1. config.json +10 -11
config.json CHANGED
@@ -1,15 +1,14 @@
1
  {
2
- "architectures": [
3
- "LDT"
4
- ],
5
  "vocab_size": 12288,
6
- "d_model": 320,
7
- "n_layers": 5,
8
- "n_heads": 5,
9
- "ffn_dim": 896,
10
- "context_length": 512,
11
  "tie_word_embeddings": true,
12
- "activation": "swiglu",
13
- "norm_eps": 1e-05,
14
- "torch_dtype": "float32"
15
  }
 
1
  {
2
+ "architectures": ["LDT"],
3
+ "model_type": "ldt",
 
4
  "vocab_size": 12288,
5
+ "hidden_size": 320,
6
+ "num_hidden_layers": 5,
7
+ "num_attention_heads": 5,
8
+ "intermediate_size": 896,
9
+ "max_position_embeddings": 512,
10
  "tie_word_embeddings": true,
11
+ "dtype": "float32",
12
+ "step": 30000,
13
+ "best_val": 3.6892
14
  }