File size: 1,650 Bytes
f011463 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | {
"architectures": [
"BaselineTransformer"
],
"model_type": "gpt2",
"auto_map": {
"AutoConfig": "modeling_transformer.TransformerConfig",
"AutoModel": "modeling_transformer.BaselineTransformer",
"AutoModelForCausalLM": "modeling_transformer.BaselineTransformer"
},
"vocab_size": 1251,
"n_positions": 256,
"n_embd": 512,
"d_model": 512,
"n_layer": 14,
"n_layers": 14,
"n_head": 8,
"n_heads": 8,
"head_dim": 64,
"n_inner": 2048,
"mlp_dim": 2048,
"max_seq_len": 256,
"dropout": 0.0,
"activation_function": "gelu",
"layer_norm_epsilon": 1e-05,
"num_parameters": 44905984,
"weight_tying": true,
"pretrain_steps": 185000,
"midtrain_steps": 10000,
"training_corpus": "v22_chinchilla_mix (740M tokens, Chinchilla-optimal)",
"midtrain_corpus": "v23_midtrain (curated high-density knowledge & dialogues)",
"author": "Thomas Seri (Sentinel)",
"project": "Sentinel R-08 Architectural Benchmark Suite",
"comparative_target": "Sentinel R-08 v7 Pyramid",
"gguf_file": "transformer-baseline-45m-f16.gguf",
"benchmarks": {
"blimp_it_pretrain": 82.53,
"blimp_it_midtrain": 85.84,
"multiblimp_pretrain": 85.15,
"multiblimp_midtrain": 86.83,
"truth_rate_pretrain": 56.2,
"truth_rate_midtrain": 68.8,
"training_throughput_tok_per_sec": 8650,
"energy_per_token_mj": 3.3,
"kv_cache_ram_at_512_mb": 5.6
},
"license": "R08-Non-Commercial-Ethical-License-v2.0",
"jurisdiction": "Court of Milan (Sezione Specializzata in Materia di Impresa), Italy",
"prohibited_uses": "Commercial, Military, Defense, Clean-Room Reverse Engineering, Model Distillation"
}
|