Hugging Face's logo

Vykos
/
Barbados

Text Generation
PyTorch
English
research
experimental
mixture-of-experts
looped-transformer
custom-code
Model card Files
xet
 Community
Barbados
File size: 4,065 Bytes
d5147ba
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
{
  "architecture": "baselines.looped_transformer@LoopedTransformer",
  "head": "lm_head@LMHead",
  "model": {
    "attn_branch_rms": "none",
    "attn_branch_rms_loops": 0,
    "attn_res": true,
    "attn_res_max_slots": 0,
    "attn_res_scope": "dual_axis",
    "attn_type": "causal",
    "bp_segment_len": 3,
    "cycle_eps_first_extra": 0.0,
    "cycle_eps_first_loops": 1,
    "cycle_eps_shared_span": 0.0,
    "cycle_eps_span_mode": "progress",
    "cycle_eps_t0": 0.0,
    "cycle_eps_t_floor": 1.0,
    "cycle_eps_time_mode": "inv_h",
    "cycle_lambda_depth_prior": 1.6,
    "cycle_lambda_depth_prior_coef": 1.0,
    "cycle_lambda_depth_prior_hi": 1.75,
    "cycle_lambda_depth_prior_lo": 1.58,
    "cycle_lambda_detach_mode": "rms7b",
    "cycle_lambda_rms_match": false,
    "cycle_residual_scale": true,
    "cycle_scale_lambda": 0.5,
    "cycle_scale_lambda_amp": 1.0,
    "cycle_scale_lambda_end": 0.0,
    "cycle_scale_lambda_learnable": false,
    "cycle_scale_lambda_max": 2.5,
    "cycle_scale_lambda_min": 1.5,
    "cycle_scale_lambda_mode": "shared",
    "cycle_scale_lambda_pack": "softplus",
    "cycle_scale_lambda_sched": "none",
    "cycle_scale_lambda_sig_eps": 0.05,
    "dual_axis_apply": "attn",
    "dual_axis_beta_h_decay": "none",
    "dual_axis_beta_h_init": 0.5,
    "dual_axis_beta_h_min": 0.0,
    "dual_axis_beta_init": 0.5,
    "dual_axis_beta_l_init": 0.5,
    "dual_axis_beta_l_min": 0.0,
    "dual_axis_beta_l_per_loop": true,
    "dual_axis_beta_learnable": false,
    "dual_axis_beta_p_init": 0.5,
    "dual_axis_blend_lambda": 0.25,
    "dual_axis_content": false,
    "dual_axis_content_mode": "exp",
    "dual_axis_content_temp": 1.0,
    "dual_axis_fuse_half": false,
    "dual_axis_fuse_mode": "sum",
    "dual_axis_h_layout": "shared",
    "dual_axis_h_update_every": 1,
    "dual_axis_h_update_src": "l",
    "dual_axis_h_write": "every_layer",
    "dual_axis_h_write_stride": 2,
    "dual_axis_heads": 1,
    "dual_axis_l_persist": false,
    "dual_axis_nd_mode": "ema",
    "dual_axis_r_scale_mode": "none",
    "dual_axis_residual_mode": "x_plus_r",
    "dual_axis_update_order": "post",
    "dual_axis_update_src": "o",
    "dual_axis_window_size": 4,
    "embed_inject_beta": 0.25,
    "embed_inject_depth_scale": true,
    "embed_inject_from_t1": false,
    "embed_inject_learnable": false,
    "embed_inject_mode": "loop_start_mix_soft",
    "embed_inject_norm": "none",
    "embed_inject_swap": false,
    "embed_inject_t1_keep": -1.0,
    "embed_inject_t1_style": "rms_x",
    "embed_inject_time_mode": "inv_t",
    "embed_inject_unit_loops": 0,
    "embed_inject_when": "start",
    "expansion": 4,
    "ffn_branch_rms": "affine",
    "ffn_type": "moe",
    "hidden_size": 384,
    "init_type": "lecun_normal",
    "layernorm_scaling": false,
    "layernorm_scaling_ell0_extra": 0.0,
    "layernorm_scaling_ell_mode": "ell",
    "layernorm_scaling_power": 0.5,
    "loop_id_embed_max": 16,
    "loop_id_embed_mode": "none",
    "loop_id_embed_scale": 1.0,
    "max_seq_len": 1023,
    "melt_enabled": true,
    "moe": {
      "capacity_factor": 1.5,
      "coarse_top_k": 1,
      "expert_granularity": 8,
      "expert_intermediate_size": 384,
      "gate_type": "sigmoid",
      "load_balance_bias_lr": 0.002,
      "num_routed_experts": 24,
      "num_shared_experts": 2,
      "router_dtype": "float32",
      "router_loop_bias": false,
      "router_num_loops": 3,
      "router_per_loop": true,
      "router_z_loss_coef": 0.001,
      "shared_expert": true,
      "shared_expert_size_multiplier": 1,
      "top_k": 6,
      "top_k_includes_shared": true
    },
    "n_layers": 6,
    "norm_eps": 1e-06,
    "norm_type": "pre",
    "num_heads": 6,
    "num_key_value_heads": 3,
    "num_loops": 3,
    "per_layer_out_norm": true,
    "pos_emb_type": "rope",
    "rope_theta": 10000.0,
    "scale_placement": "outside",
    "vocab_size": 49152
  },
  "tokenizer": {
    "eos_token_id": 0,
    "repository": "HuggingFaceTB/SmolLM-360M",
    "revision": "59f7ef243ee09a72cbc14cb054393a3e3b771d41"
  }
}