plasmova commited on
Commit
1eef4eb
·
verified ·
1 Parent(s): 70e1523

Upload Nova v3 Transformers model

Browse files
README.md ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - nova
5
+ - text-generation
6
+ - custom-code
7
+ ---
8
+
9
+ # Nova v3
10
+
11
+ Nova v3 is a decoder-only causal language model exported from the native Nova v3 checkpoint. This repository includes the model weights, tokenizer, and the custom Transformers configuration and modeling code needed to load it.
12
+
13
+ ## Model details
14
+
15
+ - Vocabulary size: 32,768
16
+ - Hidden size: 640
17
+ - Transformer layers: 26
18
+ - Attention heads: 10 query heads and 5 key/value heads
19
+ - Maximum context: 2,048 tokens
20
+ - Checkpoint step: 3,500
21
+
22
+ The exported model does not use a KV cache. Generation recomputes the full prefix at each step, so it is slower than cached Transformers models.
23
+
24
+ ## Load with Transformers
25
+
26
+ The model uses custom code in this repository. Review that code before enabling `trust_remote_code`.
27
+
28
+ ```python
29
+ import torch
30
+ from transformers import AutoModelForCausalLM, AutoTokenizer
31
+
32
+ model_id = "YOUR_USERNAME/nova-v3"
33
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
34
+ model = AutoModelForCausalLM.from_pretrained(
35
+ model_id,
36
+ trust_remote_code=True,
37
+ dtype=torch.float32,
38
+ )
39
+
40
+ inputs = tokenizer("Hello!", return_tensors="pt")
41
+ output = model.generate(
42
+ **inputs,
43
+ max_new_tokens=64,
44
+ do_sample=False,
45
+ )
46
+ print(tokenizer.decode(output[0], skip_special_tokens=True))
47
+ ```
48
+
49
+ ## Evaluation
50
+
51
+ The local v3 checkpoint scored 37.6% normalized accuracy on a zero-shot 500-example sample of HellaSwag validation. This is a limited local benchmark run, not a full validation-set result.
52
+
53
+ ## License and training data
54
+
55
+ No license or training-data statement was included with the source checkpoint. These should be added by the model publisher before public distribution.
config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "NovaForCausalLM"
4
+ ],
5
+ "dtype": "float32",
6
+ "eos_token_id": 10,
7
+ "head_dim": 64,
8
+ "hidden_size": 640,
9
+ "intermediate_size": 1920,
10
+ "max_position_embeddings": 2048,
11
+ "model_type": "nova",
12
+ "num_attention_heads": 10,
13
+ "num_hidden_layers": 26,
14
+ "num_key_value_heads": 5,
15
+ "pad_token_id": 10,
16
+ "rope_theta": 100000.0,
17
+ "transformers_version": "4.57.5",
18
+ "use_cache": false,
19
+ "vocab_size": 32768,
20
+ "auto_map": {
21
+ "AutoConfig": "configuration_nova.NovaConfig",
22
+ "AutoModelForCausalLM": "modeling_nova.NovaForCausalLM"
23
+ }
24
+ }
configuration_nova.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PretrainedConfig
2
+
3
+
4
+ class NovaConfig(PretrainedConfig):
5
+ model_type = "nova"
6
+
7
+ def __init__(
8
+ self,
9
+ vocab_size=32768,
10
+ hidden_size=640,
11
+ num_hidden_layers=26,
12
+ num_attention_heads=10,
13
+ num_key_value_heads=5,
14
+ head_dim=64,
15
+ intermediate_size=1920,
16
+ max_position_embeddings=2048,
17
+ rope_theta=100000.0,
18
+ **kwargs,
19
+ ):
20
+ super().__init__(tie_word_embeddings=True, **kwargs)
21
+ self.vocab_size = vocab_size
22
+ self.hidden_size = hidden_size
23
+ self.num_hidden_layers = num_hidden_layers
24
+ self.num_attention_heads = num_attention_heads
25
+ self.num_key_value_heads = num_key_value_heads
26
+ self.head_dim = head_dim
27
+ self.intermediate_size = intermediate_size
28
+ self.max_position_embeddings = max_position_embeddings
29
+ self.rope_theta = rope_theta
30
+
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 10,
4
+ "pad_token_id": 10,
5
+ "transformers_version": "4.57.5"
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6050e8ddaaf5e9a3aa4d3e1ab8fa8d5d04908f1a5acd9cd210f8634ef503143c
3
+ size 595243360
modeling_nova.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+ from transformers import GenerationMixin, PreTrainedModel
5
+ from transformers.modeling_outputs import CausalLMOutput
6
+
7
+ try:
8
+ from .configuration_nova import NovaConfig
9
+ except ImportError:
10
+ from configuration_nova import NovaConfig
11
+
12
+
13
+ def _rope(x, cos, sin):
14
+ a, b = x.chunk(2, dim=-1)
15
+ return torch.cat((a * cos - b * sin, b * cos + a * sin), dim=-1)
16
+
17
+
18
+ class NovaRMSNorm(nn.Module):
19
+ def __init__(self, dim):
20
+ super().__init__()
21
+ self.w = nn.Parameter(torch.ones(dim))
22
+
23
+ def forward(self, x):
24
+ return F.rms_norm(x, (x.size(-1),), self.w, 1e-6)
25
+
26
+
27
+ class NovaBlock(nn.Module):
28
+ def __init__(self, config):
29
+ super().__init__()
30
+ d, h, kv, hd, ff = (
31
+ config.hidden_size,
32
+ config.num_attention_heads,
33
+ config.num_key_value_heads,
34
+ config.head_dim,
35
+ config.intermediate_size,
36
+ )
37
+ self.h, self.kv, self.hd = h, kv, hd
38
+ self.n1, self.n2 = NovaRMSNorm(d), NovaRMSNorm(d)
39
+ self.qn, self.kn = NovaRMSNorm(hd), NovaRMSNorm(hd)
40
+ self.wq = nn.Linear(d, h * hd, bias=False)
41
+ self.wk = nn.Linear(d, kv * hd, bias=False)
42
+ self.wv = nn.Linear(d, kv * hd, bias=False)
43
+ self.wo = nn.Linear(h * hd, d, bias=False)
44
+ self.gate = nn.Linear(d, ff, bias=False)
45
+ self.up = nn.Linear(d, ff, bias=False)
46
+ self.down = nn.Linear(ff, d, bias=False)
47
+
48
+ def forward(self, x, cos, sin, attention_mask):
49
+ batch, length, _ = x.shape
50
+ h = self.n1(x)
51
+ q = self.wq(h).view(batch, length, self.h, self.hd)
52
+ k = self.wk(h).view(batch, length, self.kv, self.hd)
53
+ v = self.wv(h).view(batch, length, self.kv, self.hd)
54
+ q = _rope(self.qn(q), cos, sin).to(v.dtype).transpose(1, 2)
55
+ k = _rope(self.kn(k), cos, sin).to(v.dtype).transpose(1, 2)
56
+ v = v.transpose(1, 2)
57
+ repeat = self.h // self.kv
58
+ k = k.repeat_interleave(repeat, dim=1)
59
+ v = v.repeat_interleave(repeat, dim=1)
60
+ causal = torch.ones(length, length, dtype=torch.bool, device=x.device).tril()
61
+ mask = causal[None, None, :, :]
62
+ if attention_mask is not None:
63
+ mask = mask & attention_mask[:, None, None, :].to(torch.bool)
64
+ out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
65
+ x = x + self.wo(out.transpose(1, 2).reshape(batch, length, -1))
66
+ h = self.n2(x)
67
+ return x + self.down(F.silu(self.gate(h)) * self.up(h))
68
+
69
+
70
+ class NovaModel(PreTrainedModel):
71
+ config_class = NovaConfig
72
+ base_model_prefix = "model"
73
+ _no_split_modules = ["NovaBlock"]
74
+
75
+ def __init__(self, config):
76
+ super().__init__(config)
77
+ self.emb = nn.Embedding(config.vocab_size, config.hidden_size)
78
+ self.blocks = nn.ModuleList(NovaBlock(config) for _ in range(config.num_hidden_layers))
79
+ self.norm = NovaRMSNorm(config.hidden_size)
80
+ frequencies = torch.outer(
81
+ torch.arange(config.max_position_embeddings, dtype=torch.float32),
82
+ 1 / config.rope_theta ** (
83
+ torch.arange(0, config.head_dim, 2, dtype=torch.float32) / config.head_dim
84
+ ),
85
+ )
86
+ self.register_buffer("cos", frequencies.cos(), persistent=False)
87
+ self.register_buffer("sin", frequencies.sin(), persistent=False)
88
+ self.post_init()
89
+
90
+ def get_input_embeddings(self):
91
+ return self.emb
92
+
93
+ def set_input_embeddings(self, value):
94
+ self.emb = value
95
+
96
+ def forward(
97
+ self,
98
+ input_ids=None,
99
+ attention_mask=None,
100
+ position_ids=None,
101
+ inputs_embeds=None,
102
+ **kwargs,
103
+ ):
104
+ if inputs_embeds is None:
105
+ inputs_embeds = self.emb(input_ids)
106
+ batch, length, _ = inputs_embeds.shape
107
+ if position_ids is None:
108
+ position_ids = torch.arange(length, device=inputs_embeds.device).unsqueeze(0).expand(batch, -1)
109
+ cos = self.cos[position_ids].unsqueeze(-2)
110
+ sin = self.sin[position_ids].unsqueeze(-2)
111
+ hidden = inputs_embeds
112
+ for block in self.blocks:
113
+ hidden = block(hidden, cos, sin, attention_mask)
114
+ return self.norm(hidden)
115
+
116
+
117
+ class NovaForCausalLM(PreTrainedModel, GenerationMixin):
118
+ config_class = NovaConfig
119
+ base_model_prefix = "model"
120
+ _tied_weights_keys = []
121
+
122
+ def __init__(self, config):
123
+ super().__init__(config)
124
+ self.model = NovaModel(config)
125
+ self.post_init()
126
+
127
+ def get_input_embeddings(self):
128
+ return self.model.emb
129
+
130
+ def set_input_embeddings(self, value):
131
+ self.model.set_input_embeddings(value)
132
+
133
+ def get_output_embeddings(self):
134
+ return None
135
+
136
+ def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs):
137
+ # This model has no KV cache; each generation step recomputes the full prefix.
138
+ return {"input_ids": input_ids, "attention_mask": attention_mask}
139
+
140
+ def forward(
141
+ self,
142
+ input_ids=None,
143
+ attention_mask=None,
144
+ position_ids=None,
145
+ inputs_embeds=None,
146
+ labels=None,
147
+ **kwargs,
148
+ ):
149
+ hidden = self.model(
150
+ input_ids=input_ids,
151
+ attention_mask=attention_mask,
152
+ position_ids=position_ids,
153
+ inputs_embeds=inputs_embeds,
154
+ )
155
+ logits = F.linear(hidden, self.model.emb.weight)
156
+ loss = None
157
+ if labels is not None:
158
+ loss = F.cross_entropy(
159
+ logits[..., :-1, :].contiguous().view(-1, logits.size(-1)),
160
+ labels[..., 1:].contiguous().view(-1),
161
+ ignore_index=-100,
162
+ )
163
+ return CausalLMOutput(loss=loss, logits=logits)
164
+
special_tokens_map.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>",
3
+ "pad_token": "<|endoftext|>"
4
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|start_of_line|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|end_of_line|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<|system|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<|/system|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<|user|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "<|/user|>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "6": {
52
+ "content": "<|assistant|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "7": {
60
+ "content": "<|/assistant|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "8": {
68
+ "content": "<|think|>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "9": {
76
+ "content": "<|/think|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "10": {
84
+ "content": "<|endoftext|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ }
91
+ },
92
+ "clean_up_tokenization_spaces": false,
93
+ "eos_token": "<|endoftext|>",
94
+ "extra_special_tokens": {},
95
+ "model_max_length": 1000000000000000019884624838656,
96
+ "pad_token": "<|endoftext|>",
97
+ "tokenizer_class": "PreTrainedTokenizerFast"
98
+ }