Text Generation
English
File size: 1,180 Bytes
7051f0d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
{
 "id": "TinyChat-mix/c10_ref",
 "family": "TinyChat-mix",
 "name": "c10_ref",
 "architecture": "ReForm",
 "title": "ReForm 2+2x3 d416 (~10.2M), лучший рецепт полигона + ctx_mod",
 "params_M": 10.196,
 "tokenizer": "bpe",
 "cfg": {
  "vocab": 4096,
  "act": "relu2",
  "qk_norm": true,
  "ctx": 512,
  "dropout": 0.0,
  "ctx_ngram": 3,
  "tie": true,
  "d": 416,
  "heads": 8,
  "prelude": 2,
  "core": 2,
  "core_mix": "ca",
  "r_max": 3,
  "ponder_beta": 0.02,
  "emb_std": 0.04902903378454601,
  "think_shift": true,
  "latent_noise": 0.1,
  "lm_w": "last",
  "value_res": true,
  "depth_dense": true,
  "ctx_mod": true,
  "ctx_mod_learn": true
 },
 "training": {
  "data": "chatbpe",
  "budget_s": 600,
  "lr": 0.002,
  "batch": 32,
  "extras": {}
 },
 "metrics": {
  "val_bpc": 1.0391,
  "val_ppl": 2.055,
  "val_n_accuracy": 0.6737,
  "n_accuracy": 0.6782,
  "val_accuracy": 0.6174,
  "steps": 6745,
  "train_s": 600.1,
  "best_at_s": 580.1,
  "val_bpc_per_source": {
   "tinychat": 0.5101,
   "wiki": 1.5681
  },
  "gen_tok_s": 37.2
 },
 "prompt_example": "[INST] How do you feel today? [/INST]"
}