Hprairie commited on
Commit
b7479ec
·
verified ·
1 Parent(s): a352fbc

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. config.json +99 -0
  2. pytorch_model.bin +3 -0
  3. tokenizer.json +0 -0
  4. tokenizer_config.json +7 -0
config.json ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "parcae-large-770m",
3
+ "hf_config": {
4
+ "org": "SandyResearch",
5
+ "name": "parcae-large-770m"
6
+ },
7
+ "block_size": 2048,
8
+ "n_embd": 1280,
9
+ "intermediate_size": 5120,
10
+ "num_attention_heads": 10,
11
+ "num_key_value_heads": 10,
12
+ "vocab_size": 32768,
13
+ "padding_multiple": 64,
14
+ "padded_vocab_size": 32768,
15
+ "rope_settings": {
16
+ "use_rope": true,
17
+ "rope_condense_ratio": 1,
18
+ "rope_base": 50000
19
+ },
20
+ "use_abacus": false,
21
+ "randomize_positions_from": null,
22
+ "block_class_name": "TransformerPreNormBlock",
23
+ "norm_class_name": "RMSNorm",
24
+ "attention_class_name": "CausalSelfAttention",
25
+ "attn_impl": "flash",
26
+ "norm_eps": 1e-05,
27
+ "mlp_class_name": "BaseMLP",
28
+ "nonlin_name": "ReLU2",
29
+ "bias": false,
30
+ "qk_bias": false,
31
+ "init_strategy": "scaled-zero",
32
+ "init_orthogonal": true,
33
+ "skip_initialization": false,
34
+ "attention_nonlin_name": null,
35
+ "kv_lora_rank": null,
36
+ "q_lora_rank": null,
37
+ "qk_nope_head_dim": null,
38
+ "qk_rope_head_dim": null,
39
+ "v_head_dim": null,
40
+ "rve_rank": 1,
41
+ "rve_bias": false,
42
+ "rve_scale_by_sqrt_rank": false,
43
+ "gdn_mode": "chunk",
44
+ "gdn_expand_k": 0.75,
45
+ "gdn_expand_v": 1.5,
46
+ "gdn_qk_norm": "l2",
47
+ "gdn_conv_size": 4,
48
+ "gdn_conv_bias": false,
49
+ "gdn_gate_fn": "swish",
50
+ "gdn_gate_logit_normalizer": 16,
51
+ "gdn_fuse_norm": true,
52
+ "gdn_elementwise_affine": true,
53
+ "gdn_use_mamba_gate": true,
54
+ "gdn_use_mva": false,
55
+ "gdn_use_residual": false,
56
+ "gdn_use_input_gate": false,
57
+ "kda_mode": "chunk",
58
+ "kda_impl": "fla-chunk",
59
+ "kda_head_dim": 128,
60
+ "kda_num_v_heads": null,
61
+ "kda_expand_v": 1.0,
62
+ "kda_use_short_conv": true,
63
+ "kda_allow_neg_eigval": false,
64
+ "kda_safe_gate": false,
65
+ "kda_lower_bound": null,
66
+ "kda_disable_recompute": false,
67
+ "kda_conv_size": 4,
68
+ "kda_conv_bias": false,
69
+ "mup_model_scaling_factor": 1,
70
+ "use_fused_head": "liger",
71
+ "debias_attention": false,
72
+ "center_attention": false,
73
+ "clip_qkv": null,
74
+ "qk_norm": true,
75
+ "logit_softcap": null,
76
+ "activation_checkpoint_impl": "per-iteration",
77
+ "simple_ops": false,
78
+ "strategy": "single",
79
+ "injection_type": "diagonal",
80
+ "n_layers_in_recurrent_block": 6,
81
+ "n_layers_in_prelude": 6,
82
+ "n_layers_in_coda": 6,
83
+ "state_init": "like-init",
84
+ "recurrent_embedding_dimension": 1280,
85
+ "recurrent_intermediation_embedding_dimension": 5120,
86
+ "recurrent_num_attention_heads": null,
87
+ "prelude_norm": true,
88
+ "sampling_scheme": "poisson-truncated-full",
89
+ "recurrence_depth_variance": 12.0,
90
+ "mean_recurrence": 8,
91
+ "mean_backprop_depth": 4,
92
+ "lockstep_n": true,
93
+ "lockstep_k": false,
94
+ "curriculum_target": "forward",
95
+ "recurrent_iteration_method": "per-sequence",
96
+ "tie_embeddings": true,
97
+ "model_class_name": "Parcae",
98
+ "_is_recurrent_block_config": false
99
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1c4281b59e8db677a58d45398c1a53f6db9ac104022a6f6888b6140ad015b1b
3
+ size 3107741579
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "is_local": true,
4
+ "local_files_only": false,
5
+ "model_max_length": 1000000000000000019884624838656,
6
+ "tokenizer_class": "TokenizersBackend"
7
+ }