Hugging Face's logo

KitsuVp
/
NeoLLM

Text Generation
Transformers
Safetensors
PyTorch
English
neollm
causal-lm
research
bf16
attention
normalization
conversational
custom_code
Model card Files
xet
 Community
2
KitsuVp commited on
Commit
f82204d
·
verified ·
1 Parent(s): 728809f

Model save

Browse files
Files changed (4) hide show
  1. README.md +7 -13
  2. config.json +25 -3
  3. model.safetensors +2 -2
  4. training_args.bin +1 -1
README.md CHANGED
@@ -14,7 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
14
 
15
  This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
16
  It achieves the following results on the evaluation set:
17
- - Loss: 3.3061
18
 
19
  ## Model description
20
 
@@ -44,22 +44,16 @@ The following hyperparameters were used during training:
44
 
45
  ### Training results
46
 
47
- | Training Loss | Epoch | Step | Validation Loss |
48
- |:-------------:|:------:|:-----:|:---------------:|
49
- | 3.8671 | 0.1067 | 5000 | 3.7862 |
50
- | 3.6710 | 0.2133 | 10000 | 3.5860 |
51
- | 3.5866 | 0.32 | 15000 | 3.5085 |
52
- | 3.5398 | 0.4267 | 20000 | 3.4550 |
53
- | 3.5042 | 0.5333 | 25000 | 3.4161 |
54
- | 3.4765 | 0.64 | 30000 | 3.3908 |
55
- | 3.4618 | 0.7467 | 35000 | 3.3693 |
56
- | 3.4085 | 0.8533 | 40000 | 3.3357 |
57
- | 3.3873 | 0.96 | 45000 | 3.3061 |
58
 
59
 
60
  ### Framework versions
61
 
62
  - Transformers 5.3.0
63
  - Pytorch 2.10.0+cu130
64
- - Datasets 4.7.0
65
  - Tokenizers 0.22.2
 
14
 
15
  This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
16
  It achieves the following results on the evaluation set:
17
+ - Loss: 2.9227
18
 
19
  ## Model description
20
 
 
44
 
45
  ### Training results
46
 
47
+ | Training Loss | Epoch | Step | Validation Loss |
48
+ |:-------------:|:-----:|:-----:|:---------------:|
49
+ | 3.7355 | 0.32 | 5000 | 3.6867 |
50
+ | 3.2793 | 0.64 | 10000 | 3.2132 |
51
+ | 3.0157 | 0.96 | 15000 | 2.9227 |
 
 
 
 
 
 
52
 
53
 
54
  ### Framework versions
55
 
56
  - Transformers 5.3.0
57
  - Pytorch 2.10.0+cu130
58
+ - Datasets 4.8.2
59
  - Tokenizers 0.22.2
config.json CHANGED
@@ -1,33 +1,48 @@
1
  {
 
2
  "architectures": [
3
  "NeoLLMForCausalLM"
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.1,
 
7
  "auto_map": {
8
  "AutoConfig": "configuration_neollm.NeoLLMConfig",
9
  "AutoModel": "modeling_neollm.NeoLLMModel",
10
  "AutoModelForCausalLM": "modeling_neollm.NeoLLMForCausalLM"
11
  },
 
 
12
  "dropout_rate": 0.1,
13
  "dtype": "bfloat16",
14
  "eos_token_id": 151643,
15
  "fan_ratio": 0.125,
16
  "fan_ratio_ffn": 0.0625,
 
 
 
 
 
 
17
  "head_dim": 64,
18
  "hidden_act": "xielu",
19
  "hidden_size": 512,
20
  "initializer_range": 0.02,
21
  "intermediate_size": 1536,
 
 
 
 
 
22
  "lucid_attention_eps": 1e-06,
23
  "max_position_embeddings": 512,
24
- "mea_component_key_value_heads": 4,
25
  "mea_groupnorm_eps": 1e-06,
26
  "model_type": "neollm",
27
  "momentum_gamma": 0.1,
28
  "num_attention_heads": 8,
29
  "num_hidden_layers": 12,
30
- "num_key_value_heads": 4,
31
  "pad_token_id": 151643,
32
  "partial_rotary_factor": 0.25,
33
  "rms_norm_eps": 1e-06,
@@ -39,9 +54,16 @@
39
  "rope_theta": 10000.0,
40
  "tie_word_embeddings": true,
41
  "transformers_version": "5.3.0",
 
 
42
  "use_cache": false,
 
 
43
  "use_lucid_attention": true,
44
  "use_mea_attention": true,
45
  "use_momentum_attention": true,
46
- "vocab_size": 151665
 
 
 
47
  }
 
1
  {
2
+ "affine_momentum": 0.9,
3
  "architectures": [
4
  "NeoLLMForCausalLM"
5
  ],
6
  "attention_bias": false,
7
  "attention_dropout": 0.1,
8
+ "attn_res_num_blocks": 4,
9
  "auto_map": {
10
  "AutoConfig": "configuration_neollm.NeoLLMConfig",
11
  "AutoModel": "modeling_neollm.NeoLLMModel",
12
  "AutoModelForCausalLM": "modeling_neollm.NeoLLMForCausalLM"
13
  },
14
+ "directional_routing_k": 4,
15
+ "directional_routing_temp": 5.0,
16
  "dropout_rate": 0.1,
17
  "dtype": "bfloat16",
18
  "eos_token_id": 151643,
19
  "fan_ratio": 0.125,
20
  "fan_ratio_ffn": 0.0625,
21
+ "generator_d_seed": 128,
22
+ "generator_k": 3,
23
+ "generator_krank": 16,
24
+ "generator_num_knots": 32,
25
+ "generator_num_modes": 8,
26
+ "generator_spline_degree": 2,
27
  "head_dim": 64,
28
  "hidden_act": "xielu",
29
  "hidden_size": 512,
30
  "initializer_range": 0.02,
31
  "intermediate_size": 1536,
32
+ "jtokm_aux_loss_weight": 0.0001,
33
+ "jtokm_norm_eps": 1e-06,
34
+ "jtokm_num_experts": 4,
35
+ "jtokm_num_modes": 4,
36
+ "jtokm_top_k": 2,
37
  "lucid_attention_eps": 1e-06,
38
  "max_position_embeddings": 512,
39
+ "mea_component_key_value_heads": 2,
40
  "mea_groupnorm_eps": 1e-06,
41
  "model_type": "neollm",
42
  "momentum_gamma": 0.1,
43
  "num_attention_heads": 8,
44
  "num_hidden_layers": 12,
45
+ "num_key_value_heads": 2,
46
  "pad_token_id": 151643,
47
  "partial_rotary_factor": 0.25,
48
  "rms_norm_eps": 1e-06,
 
54
  "rope_theta": 10000.0,
55
  "tie_word_embeddings": true,
56
  "transformers_version": "5.3.0",
57
+ "use_affine_scaled_attention": true,
58
+ "use_attn_res": true,
59
  "use_cache": false,
60
+ "use_directional_routing": true,
61
+ "use_jtokm": false,
62
  "use_lucid_attention": true,
63
  "use_mea_attention": true,
64
  "use_momentum_attention": true,
65
+ "use_token_generator": false,
66
+ "use_xsa": true,
67
+ "vocab_size": 151665,
68
+ "xsa_eps": 1e-06
69
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b86e70464eac60bf2e31c9bb80ecf49143001756ffde78208af382ee926c3960
3
- size 254583392
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fbf1e058bbc5382aee6ea8d911ef8feda92bec95359b39edafe711b225fbaf47
3
+ size 270562344
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e21b9bd176ce940ed56dc325927aefec7471cb0899dffccbbcfcc9774a5ea9a2
3
  size 5329
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44c8cf837bfc91e52399e6833deb3364d7dcc6086f3a479e07fa02da20ed5fe2
3
  size 5329