Hugging Face's logo

chameleon-lizard
/
buckwheat-diffusion-base

Safetensors
PyTorch
diffusion
masked-language-modeling
Model card Files
xet
 Community
buckwheat-diffusion-base
File size: 1,070 Bytes
f757d4c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
{
  "kind": "diffusion",
  "model": {
    "vocab_size": 50368,
    "max_seq_len": 2048,
    "d_model": 768,
    "n_layers": 22,
    "n_heads": 12,
    "d_ff": 1152,
    "dropout": 0.0,
    "embedding_dropout": 0.0,
    "mlp_dropout": 0.0,
    "attention_dropout": 0.0,
    "attention_bias": false,
    "mlp_bias": false,
    "norm_bias": false,
    "prediction_bias": false,
    "decoder_bias": true,
    "norm_eps": 1e-05,
    "local_attention": 128,
    "global_attn_every_n_layers": 3,
    "local_rope_theta": 160000.0,
    "global_rope_theta": 160000.0,
    "initializer_range": 0.02,
    "initializer_cutoff_factor": 2.0,
    "pad_token_id": 50283,
    "unk_token_id": 50280,
    "bos_token_id": 50281,
    "eos_token_id": 50282,
    "mask_token_id": 50284,
    "label_ignore_index": -100,
    "mlm_probability": 0.15,
    "diffusion_steps": 1000,
    "do_not_mask_token_ids": [
      50279,
      50280,
      50281,
      50282,
      50283,
      50284
    ]
  },
  "training": {
    "rope_fp32": true
  },
  "format": "pretrain",
  "tokens_seen": 56998518784
}