Hugging Face's logo

N8Programs
/
BestTerm-440M-Checkpts

Text Generation
Transformers
Safetensors
qwen3
oeis
integer-sequences
sequence-modeling
slerp
text-generation-inference
Model card Files
xet
 Community
N8Programs commited on
Commit
136c9c9
·
verified ·
1 Parent(s): 3a203f1

Upload tentative BestTerm-440M checkpoint

Browse files

Tentative BestTerm-440M: global SLERP t=0.80 between N8Programs/NextTerm-440M base and hot b-file-only 500M-token checkpoint.

Files changed (7) hide show
  1. config.json +65 -0
  2. generation_config.json +8 -0
  3. model.safetensors +3 -0
  4. slerp_meta.json +13 -0
  5. special_tokens_map.json +9 -0
  6. tokenizer.json +159 -0
  7. tokenizer_config.json +47 -0
config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 12,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 13,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 1024,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 3072,
15
+ "layer_types": [
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention",
43
+ "full_attention"
44
+ ],
45
+ "max_position_embeddings": 40960,
46
+ "max_window_layers": 28,
47
+ "model_type": "qwen3",
48
+ "num_attention_heads": 16,
49
+ "num_hidden_layers": 28,
50
+ "num_key_value_heads": 8,
51
+ "pad_token_id": 14,
52
+ "rms_norm_eps": 1e-06,
53
+ "rope_parameters": {
54
+ "rope_theta": 1000000.0,
55
+ "rope_type": "default"
56
+ },
57
+ "sliding_window": null,
58
+ "tie_word_embeddings": false,
59
+ "transformers_version": "5.9.0",
60
+ "use_cache": true,
61
+ "use_sliding_window": false,
62
+ "vocab_size": 16,
63
+ "rope_theta": 1000000.0,
64
+ "torch_dtype": "bfloat16"
65
+ }
generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 12,
3
+ "do_sample": false,
4
+ "eos_token_id": 13,
5
+ "pad_token_id": 14,
6
+ "transformers_version": "5.9.0",
7
+ "use_cache": true
8
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6a4244107d621c7138060d37168345b9e83a2e8036ccf4ccbcd465b048068f2
3
+ size 881035528
slerp_meta.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "merge": "global_slerp",
3
+ "t": 0.8,
4
+ "base": "/workspace/nextterm_bfile_posttrain/base_model",
5
+ "other": "/workspace/nextterm_bfile_posttrain/runs/bfile_only_1epoch_hotlr/hot500_hf_export",
6
+ "floating_parameters": 440500224,
7
+ "global_cosine": 0.9830955502823407,
8
+ "coeff_base": 0.2010891797241661,
9
+ "coeff_other": 0.8016321071416131,
10
+ "norm_base": 170366075.61881256,
11
+ "norm_other": 179761175.357687,
12
+ "dot": 172042299.90441895
13
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<unused>"
4
+ ],
5
+ "bos_token": "<bos>",
6
+ "eos_token": "<eos>",
7
+ "pad_token": "<pad>",
8
+ "unk_token": "<pad>"
9
+ }
tokenizer.json ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 12,
8
+ "content": "<bos>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 13,
17
+ "content": "<eos>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 14,
26
+ "content": "<pad>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 15,
35
+ "content": "<unused>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ }
42
+ ],
43
+ "normalizer": {
44
+ "type": "Sequence",
45
+ "normalizers": [
46
+ {
47
+ "type": "Replace",
48
+ "pattern": {
49
+ "Regex": "[^0-9,-]+"
50
+ },
51
+ "content": ""
52
+ },
53
+ {
54
+ "type": "Replace",
55
+ "pattern": {
56
+ "Regex": ",+"
57
+ },
58
+ "content": ","
59
+ },
60
+ {
61
+ "type": "Strip",
62
+ "strip_left": true,
63
+ "strip_right": true
64
+ },
65
+ {
66
+ "type": "Replace",
67
+ "pattern": {
68
+ "Regex": "^,+"
69
+ },
70
+ "content": ""
71
+ },
72
+ { "type": "Replace", "pattern": { "Regex": ",{2,}$" }, "content": "," }
73
+ ]
74
+ },
75
+ "pre_tokenizer": {
76
+ "type": "Split",
77
+ "pattern": {
78
+ "Regex": ""
79
+ },
80
+ "behavior": "Isolated",
81
+ "invert": false
82
+ },
83
+ "post_processor": {
84
+ "type": "TemplateProcessing",
85
+ "single": [
86
+ {
87
+ "SpecialToken": {
88
+ "id": "<bos>",
89
+ "type_id": 0
90
+ }
91
+ },
92
+ {
93
+ "Sequence": {
94
+ "id": "A",
95
+ "type_id": 0
96
+ }
97
+ }
98
+ ],
99
+ "pair": [
100
+ {
101
+ "Sequence": {
102
+ "id": "A",
103
+ "type_id": 0
104
+ }
105
+ },
106
+ {
107
+ "Sequence": {
108
+ "id": "B",
109
+ "type_id": 1
110
+ }
111
+ }
112
+ ],
113
+ "special_tokens": {
114
+ "<bos>": {
115
+ "id": "<bos>",
116
+ "ids": [
117
+ 12
118
+ ],
119
+ "tokens": [
120
+ "<bos>"
121
+ ]
122
+ }
123
+ }
124
+ },
125
+ "decoder": {
126
+ "type": "Sequence",
127
+ "decoders": [
128
+ {
129
+ "type": "Replace",
130
+ "pattern": {
131
+ "String": " "
132
+ },
133
+ "content": ""
134
+ }
135
+ ]
136
+ },
137
+ "model": {
138
+ "type": "WordLevel",
139
+ "vocab": {
140
+ "0": 0,
141
+ "1": 1,
142
+ "2": 2,
143
+ "3": 3,
144
+ "4": 4,
145
+ "5": 5,
146
+ "6": 6,
147
+ "7": 7,
148
+ "8": 8,
149
+ "9": 9,
150
+ "-": 10,
151
+ ",": 11,
152
+ "<bos>": 12,
153
+ "<eos>": 13,
154
+ "<pad>": 14,
155
+ "<unused>": 15
156
+ },
157
+ "unk_token": "<pad>"
158
+ }
159
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "12": {
4
+ "content": "<bos>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "13": {
12
+ "content": "<eos>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "14": {
20
+ "content": "<pad>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "15": {
28
+ "content": "<unused>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ }
35
+ },
36
+ "additional_special_tokens": [
37
+ "<unused>"
38
+ ],
39
+ "bos_token": "<bos>",
40
+ "clean_up_tokenization_spaces": false,
41
+ "eos_token": "<eos>",
42
+ "extra_special_tokens": {},
43
+ "model_max_length": 40960,
44
+ "pad_token": "<pad>",
45
+ "tokenizer_class": "PreTrainedTokenizerFast",
46
+ "unk_token": "<pad>"
47
+ }