Hugging Face's logo

IFM
/
LoopedLM-P1-dense-split-hl-336b

Text Generation
Safetensors
xllm
looped-language-model
paper-part1
dense
Model card Files
xet
 Community
LoopedLM-P1-dense-split-hl-336b
File size: 2,394 Bytes
2cdd3ba
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
{
  "model": {
    "apply_attn_gate": false,
    "apply_bias_term": true,
    "apply_rmsnorm": true,
    "arch": "huginn",
    "attention_dropout": 0.0,
    "attn_gate_func": "silu",
    "causal_attn_backend": "flash",
    "chunk_size": 1024,
    "dense_prelude_diagonal_a_init": 1.0,
    "dense_prelude_diagonal_dt_init": 1.0,
    "dense_prelude_input_injection": "none",
    "dense_prelude_input_layers": "",
    "dense_prelude_source_layer": -1,
    "dropout": 0.0,
    "expert_inter_dim": 0,
    "ffn_hidden_dim": 4352,
    "fused_block": true,
    "fused_output_layer": true,
    "head_dim": 64,
    "hidden_dropout": 0.0,
    "huginn_coda_layers": 8,
    "huginn_diagonal_a_init": 1.0,
    "huginn_diagonal_dt_init": 1.0,
    "huginn_hierarchical_h_cycles": 2,
    "huginn_hierarchical_l_cycles": 3,
    "huginn_hierarchical_state": "split_hl",
    "huginn_input_injection": "diagonal",
    "huginn_prelude_layers": 8,
    "huginn_recurrent_layers": 12,
    "huginn_state_init": "normal",
    "init_embed_std": null,
    "init_logits_std": null,
    "init_mode": "gaussian",
    "init_std": 0.02,
    "layernorm_num_groups": 1,
    "layerwise_ckpt": false,
    "loop_diagonal_a_init": 1.0,
    "loop_diagonal_dt_init": 1.0,
    "loop_end_layers": 28,
    "loop_input_injection": "none",
    "loop_start_layer": 0,
    "loop_times": 8,
    "memory_efficient_norm": false,
    "model_dim": 1536,
    "moe_expert_backend": "sequential",
    "moe_router_bias": false,
    "moe_router_bias_update_rate": null,
    "moe_router_scaling_factor": null,
    "moe_router_score_func": "sigmoid",
    "norm_affine": true,
    "num_activated_experts": 0,
    "num_activated_values": 0,
    "num_dense_layers": null,
    "num_experts": 0,
    "num_heads": 24,
    "num_kv_heads": 6,
    "num_layers": 28,
    "num_shared_experts": 0,
    "num_values": 0,
    "output_size": -1,
    "qknorm": false,
    "recompute_attention": true,
    "recompute_fc1_out": true,
    "recompute_fc3_out": true,
    "recompute_logits": false,
    "recompute_q": true,
    "recompute_router": false,
    "recompute_v": true,
    "rmsnorm_eps": 1e-06,
    "rope_base": 1000000.0,
    "rope_head_dim": 64,
    "scale_emb": false,
    "swiglu": true,
    "vocab_size": 250624
  },
  "tokenizer": {
    "num_reserved_special_tokens": 0,
    "path": "tokenizer",
    "template_dict_json": null,
    "type": "huggingface"
  }
}