GGUFGuy commited on
Commit
2fa9fef
·
verified ·
1 Parent(s): 94ebe5c

Replace Novi-Nano-Base v1 with v2

Browse files
README.md ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - novi
5
+ - novi-nano
6
+ - causal-lm
7
+ - text-generation
8
+ - from-scratch
9
+ ---
10
+
11
+ # Novi-Nano-Base
12
+
13
+ Novi-Nano-Base v2.
14
+
15
+ A tiny causal language model trained from scratch by Novi-AI.
16
+
17
+ ## Architecture
18
+
19
+ - Vocabulary: 8,192
20
+ - Context length: 256
21
+ - Embedding size: 96
22
+ - Layers: 4
23
+ - Attention heads: 4
24
+ - FFN size: 384
25
+ - Parameters: 1,258,560
26
+
27
+ ## Training
28
+
29
+ - Training tokens: 300,023,808
30
+ - Final validation loss: 5.418699
31
+ - Final validation perplexity: 225.5853
32
+ - Best validation loss: 5.418699
33
+
34
+ This version replaces the previous Novi-Nano-Base v1 model.
config.json CHANGED
@@ -5,18 +5,18 @@
5
  "GPT2LMHeadModel"
6
  ],
7
  "attn_pdrop": 0.0,
8
- "bos_token_id": 0,
9
  "dtype": "float32",
10
  "embd_pdrop": 0.0,
11
- "eos_token_id": 0,
12
  "initializer_range": 0.02,
13
  "layer_norm_epsilon": 1e-05,
14
  "model_type": "gpt2",
15
  "n_ctx": 256,
16
- "n_embd": 32,
17
- "n_head": 2,
18
- "n_inner": 128,
19
- "n_layer": 2,
20
  "n_positions": 256,
21
  "pad_token_id": null,
22
  "reorder_and_upcast_attn": false,
@@ -30,6 +30,6 @@
30
  "summary_use_proj": true,
31
  "tie_word_embeddings": true,
32
  "transformers_version": "5.16.1",
33
- "use_cache": true,
34
- "vocab_size": 30720
35
  }
 
5
  "GPT2LMHeadModel"
6
  ],
7
  "attn_pdrop": 0.0,
8
+ "bos_token_id": 1,
9
  "dtype": "float32",
10
  "embd_pdrop": 0.0,
11
+ "eos_token_id": 2,
12
  "initializer_range": 0.02,
13
  "layer_norm_epsilon": 1e-05,
14
  "model_type": "gpt2",
15
  "n_ctx": 256,
16
+ "n_embd": 96,
17
+ "n_head": 4,
18
+ "n_inner": 384,
19
+ "n_layer": 4,
20
  "n_positions": 256,
21
  "pad_token_id": null,
22
  "reorder_and_upcast_attn": false,
 
30
  "summary_use_proj": true,
31
  "tie_word_embeddings": true,
32
  "transformers_version": "5.16.1",
33
+ "use_cache": false,
34
+ "vocab_size": 8192
35
  }
generation_config.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "_from_model_config": true,
3
- "bos_token_id": 0,
4
- "eos_token_id": 0,
5
  "output_attentions": false,
6
  "output_hidden_states": false,
7
  "transformers_version": "5.16.1",
 
1
  {
2
  "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
  "output_attentions": false,
6
  "output_hidden_states": false,
7
  "transformers_version": "5.16.1",
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c518ca87972ad4179678eb46e4a2f118c6e479c0d0e9772b87c2bc583e4f9de6
3
- size 4069416
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:782e46129b1700948bbed1db2994bc1a0fadacca2f1366835a8eaf610561f56a
3
+ size 5039208
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json DELETED
@@ -1,8 +0,0 @@
1
- {
2
- "backend": "tokenizers",
3
- "bos_token": "<|endoftext|>",
4
- "eos_token": "<|endoftext|>",
5
- "model_max_length": 1000000000000000019884624838656,
6
- "pad_token": "<|endoftext|>",
7
- "tokenizer_class": "TokenizersBackend"
8
- }
 
 
 
 
 
 
 
 
 
training_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Novi-Nano-Base",
3
+ "version": 2,
4
+ "vocab_size": 8192,
5
+ "block_size": 256,
6
+ "n_embd": 96,
7
+ "n_layer": 4,
8
+ "n_head": 4,
9
+ "n_inner": 384,
10
+ "parameters": 1258560,
11
+ "training_tokens": 300000000,
12
+ "tokens_seen": 300023808,
13
+ "optimizer_steps": 2289,
14
+ "batch_size": 256,
15
+ "grad_accum_steps": 2,
16
+ "learning_rate": 0.0003,
17
+ "min_learning_rate": 3e-05,
18
+ "warmup_steps": 500,
19
+ "weight_decay": 0.1,
20
+ "grad_clip": 1.0,
21
+ "validation_tokens": 5000000,
22
+ "final_validation_loss": 5.418698577137737,
23
+ "final_validation_perplexity": 225.5853494508474,
24
+ "best_validation_loss": 5.418698577137737
25
+ }
training_metadata.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "model_name": "Novi-Nano-Base",
3
- "parameters": 1016704,
4
- "vocab_size": 30720,
5
- "block_size": 256,
6
- "n_embd": 32,
7
- "n_layer": 2,
8
- "n_head": 2,
9
- "n_inner": 128,
10
- "training_tokens": 300023808,
11
- "target_training_tokens": 300000000,
12
- "validation_tokens": 5000000,
13
- "validation_is_shared_dataset_stream": true,
14
- "final_validation_loss": 6.641223424440854,
15
- "final_validation_perplexity": 766.0316017522573,
16
- "best_validation_loss": 6.641223424440854,
17
- "optimizer_steps": 2289,
18
- "learning_rate": 0.0003,
19
- "minimum_learning_rate": 3e-05,
20
- "weight_decay": 0.1,
21
- "warmup_steps": 500,
22
- "seed": 42
23
- }