Hugging Face's logo

aklein4
/
vae-init

Model card Files
xet
 Community
aklein4 commited on
Commit
6f84ad2
·
verified ·
1 Parent(s): 82f9836

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. 000000000000/config.json +149 -0
  2. 000000000000/model.pt +3 -0
000000000000/config.json ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "vae.VAEModel",
3
+ "pretrained_url": null,
4
+ "pretrained_step": null,
5
+ "pretrained_strict": null,
6
+ "torch_dtype": "float32",
7
+ "vocab_size": 128256,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": 128001,
10
+ "pad_token_id": -100,
11
+ "hidden_size": 2048,
12
+ "num_hidden_layers": 16,
13
+ "num_attention_heads": 32,
14
+ "num_key_value_heads": 8,
15
+ "intermediate_size": 8192,
16
+ "hidden_act": "silu",
17
+ "max_position_embeddings": 131072,
18
+ "rope_theta": 500000.0,
19
+ "rope_scaling": {
20
+ "factor": 32.0,
21
+ "high_freq_factor": 4.0,
22
+ "low_freq_factor": 1.0,
23
+ "original_context_len": 8192
24
+ },
25
+ "initializer_range": 0.02,
26
+ "attention_dropout": false,
27
+ "attention_bias": false,
28
+ "rms_norm_eps": 1e-05,
29
+ "attention_kernel": "flash_attention",
30
+ "pure_modules": [],
31
+ "sharding": {
32
+ "replicate_default": true,
33
+ "encoder_causal_layers.layers.*.mlp.gate_proj.weight": [
34
+ "fsdp",
35
+ null
36
+ ],
37
+ "encoder_causal_layers.layers.*.mlp.up_proj.weight": [
38
+ "fsdp",
39
+ null
40
+ ],
41
+ "encoder_causal_layers.layers.*.mlp.down_proj.weight": [
42
+ "fsdp",
43
+ null
44
+ ],
45
+ "encoder_noncausal.layers.layers.*.mlp.gate_proj.weight": [
46
+ "fsdp",
47
+ null
48
+ ],
49
+ "encoder_noncausal.layers.layers.*.mlp.up_proj.weight": [
50
+ "fsdp",
51
+ null
52
+ ],
53
+ "encoder_noncausal.layers.layers.*.mlp.down_proj.weight": [
54
+ "fsdp",
55
+ null
56
+ ],
57
+ "decoder_layers.layers.*.mlp.gate_proj.weight": [
58
+ "fsdp",
59
+ null
60
+ ],
61
+ "decoder_layers.layers.*.mlp.up_proj.weight": [
62
+ "fsdp",
63
+ null
64
+ ],
65
+ "decoder_layers.layers.*.mlp.down_proj.weight": [
66
+ "fsdp",
67
+ null
68
+ ],
69
+ "encoder_causal_layers.layers.*": [
70
+ [
71
+ "data",
72
+ "fsdp"
73
+ ],
74
+ null,
75
+ null
76
+ ],
77
+ "encoder_noncausal.layers.layers.*": [
78
+ [
79
+ "data",
80
+ "fsdp"
81
+ ],
82
+ null,
83
+ null
84
+ ],
85
+ "decoder_layers.layers.*": [
86
+ [
87
+ "data",
88
+ "fsdp"
89
+ ],
90
+ null,
91
+ null
92
+ ]
93
+ },
94
+ "remat": {
95
+ "advanced": [
96
+ {
97
+ "name": "encoder_causal_layers",
98
+ "settings": {
99
+ "activation_checkpoint_layers": [
100
+ "VAEEncoderCausalLayer"
101
+ ],
102
+ "optimization_barrier_layers": [
103
+ "VAEEncoderCausalLayer"
104
+ ],
105
+ "scan_layers": "layers",
106
+ "is_layer_pure": true
107
+ }
108
+ },
109
+ {
110
+ "name": "encoder_noncausal",
111
+ "settings": {
112
+ "activation_checkpoint_layers": [
113
+ "BidirectionalDecoderLayer"
114
+ ],
115
+ "optimization_barrier_layers": [
116
+ "BidirectionalDecoderLayer"
117
+ ],
118
+ "scan_layers": "layers.layers",
119
+ "is_layer_pure": true
120
+ }
121
+ },
122
+ {
123
+ "name": "decoder_layers",
124
+ "settings": {
125
+ "activation_checkpoint_layers": [
126
+ "VAEDecoderLayer"
127
+ ],
128
+ "optimization_barrier_layers": [
129
+ "VAEDecoderLayer"
130
+ ],
131
+ "scan_layers": "layers",
132
+ "is_layer_pure": true
133
+ }
134
+ }
135
+ ]
136
+ },
137
+ "num_encoder_causal_layers": 10,
138
+ "num_noncausal_layers": 6,
139
+ "num_decoder_layers": 16,
140
+ "latent_size": 2048,
141
+ "num_latent_key_heads": 8,
142
+ "num_latent_value_heads": 64,
143
+ "init_mse_lambda": 0.1,
144
+ "radius_min": 0.0,
145
+ "radius_max": 1.0,
146
+ "latent_rms_norm_eps": 1e-12,
147
+ "mixer_kernel_size": 5,
148
+ "pad_attention_bias_value": -1000.0
149
+ }
000000000000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6816ade5b1bdeda12f0cd5726d8559ba66b91ab72b688e3bcb4a9a5d04357ff7
3
+ size 11414947651