Hugging Face's logo

bimabk
/
dev-S1d

Safetensors
zamba2
Model card Files
xet
 Community
bimabk commited on
Commit
2162135
·
verified ·
1 Parent(s): dfdbcd8

Upload task output 852138c5-fa09-4153-8759-e1a93bc5ef10

Browse files
Files changed (6) hide show
  1. config.json +103 -0
  2. generation_config.json +9 -0
  3. loss.txt +1 -0
  4. model.safetensors +3 -0
  5. tokenizer.json +0 -0
  6. tokenizer_config.json +17 -0
config.json ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "adapter_rank": 128,
3
+ "add_bias_linear": false,
4
+ "architectures": [
5
+ "Zamba2ForCausalLM"
6
+ ],
7
+ "attention_dropout": 0.0,
8
+ "attention_head_dim": 128,
9
+ "attention_hidden_size": 4096,
10
+ "bos_token_id": 1,
11
+ "chunk_size": 256,
12
+ "dtype": "bfloat16",
13
+ "eos_token_id": 2,
14
+ "ffn_hidden_size": 8192,
15
+ "hidden_act": "gelu",
16
+ "hidden_size": 2048,
17
+ "hybrid_layer_ids": [
18
+ 5,
19
+ 11,
20
+ 17,
21
+ 23,
22
+ 29,
23
+ 35
24
+ ],
25
+ "initializer_range": 0.02,
26
+ "intermediate_size": 8192,
27
+ "kv_channels": 64,
28
+ "layers_block_type": [
29
+ "mamba",
30
+ "mamba",
31
+ "mamba",
32
+ "mamba",
33
+ "mamba",
34
+ "hybrid",
35
+ "mamba",
36
+ "mamba",
37
+ "mamba",
38
+ "mamba",
39
+ "mamba",
40
+ "hybrid",
41
+ "mamba",
42
+ "mamba",
43
+ "mamba",
44
+ "mamba",
45
+ "mamba",
46
+ "hybrid",
47
+ "mamba",
48
+ "mamba",
49
+ "mamba",
50
+ "mamba",
51
+ "mamba",
52
+ "hybrid",
53
+ "mamba",
54
+ "mamba",
55
+ "mamba",
56
+ "mamba",
57
+ "mamba",
58
+ "hybrid",
59
+ "mamba",
60
+ "mamba",
61
+ "mamba",
62
+ "mamba",
63
+ "mamba",
64
+ "hybrid",
65
+ "mamba",
66
+ "mamba"
67
+ ],
68
+ "mamba_d_conv": 4,
69
+ "mamba_d_state": 128,
70
+ "mamba_expand": 2,
71
+ "mamba_headdim": 64,
72
+ "mamba_ngroups": 1,
73
+ "max_position_embeddings": 4096,
74
+ "model_type": "zamba2",
75
+ "n_mamba_heads": 64,
76
+ "num_attention_heads": 32,
77
+ "num_hidden_layers": 38,
78
+ "num_key_value_heads": 32,
79
+ "num_logits_to_keep": 1,
80
+ "num_mem_blocks": 1,
81
+ "num_query_groups": 32,
82
+ "pad_token_id": 0,
83
+ "rms_norm_eps": 1e-05,
84
+ "rope_parameters": {
85
+ "rope_theta": 10000,
86
+ "rope_type": "default"
87
+ },
88
+ "tie_word_embeddings": true,
89
+ "time_step_floor": 0.0001,
90
+ "time_step_limit": null,
91
+ "time_step_max": 0.1,
92
+ "time_step_min": 0.001,
93
+ "transformers_version": "5.12.1",
94
+ "use_cache": false,
95
+ "use_conv_bias": true,
96
+ "use_long_context": false,
97
+ "use_mamba_kernels": true,
98
+ "use_mem_eff_path": false,
99
+ "use_mem_rope": true,
100
+ "use_shared_attention_adapter": true,
101
+ "use_shared_mlp_adapter": true,
102
+ "vocab_size": 32000
103
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": [
5
+ 2
6
+ ],
7
+ "pad_token_id": 0,
8
+ "transformers_version": "5.12.1"
9
+ }
loss.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 99,no_eval
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ddbaf38d2f9ac1840eaa65aab286f2c8d1c957a95c0e6f15676c3abb55dc9bc
3
+ size 2430175896
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "extra_special_tokens": [],
8
+ "is_local": true,
9
+ "local_files_only": false,
10
+ "model_max_length": 1000000000000000019884624838656,
11
+ "pad_token": "[PAD]",
12
+ "sp_model_kwargs": {},
13
+ "spaces_between_special_tokens": false,
14
+ "tokenizer_class": "LlamaTokenizer",
15
+ "unk_token": null,
16
+ "use_default_system_prompt": false
17
+ }