File size: 4,638 Bytes
fee196d f82204d fee196d a29f88e 85da451 15d7352 9a7ec7b dfb3337 9a7ec7b ca90ed0 3910163 2c9bde9 f82204d 3562d53 85da451 fee196d d341641 ca90ed0 fee196d faffbea f82204d 431dacf 0bf0772 f82204d fee196d 8cd30bb 4f4acde be1c851 fee196d 9a7ec7b 5e2ad59 377c964 5e2ad59 2d81c76 3910163 433336a fee196d be1c851 433336a f61e925 3d4901c f61e925 fee196d 433336a f61e925 3910163 a442517 4053a08 b58ca6a 3910163 fee196d be1c851 6dc415d ca90ed0 fee196d 84f1633 3f44838 713f0fb c2e2271 3f44838 fee196d 91342c4 fee196d e975e7d 374a1f9 2c9bde9 22ede6a d341641 6dc415d be1c851 223a19e 3d28735 21fc130 8059a12 91342c4 f34ab4c 2c9bde9 ca90ed0 75f871d be1c851 e975e7d 24bf54c 8cd30bb 3d4901c 2c9bde9 ca90ed0 72e7349 3910163 d227ba4 0185118 72e7349 70d15a3 f61e925 433336a f61e925 3910163 4a7401c f61e925 2c9bde9 892a8c3 21fc130 f3e3277 374a1f9 8073f81 d341641 8073f81 2c9bde9 3d28735 f34ab4c f82204d ca90ed0 7c52c18 fee196d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | {
"affine_momentum": 0.9,
"architectures": [
"NeoLLMForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.1,
"attn_res_num_blocks": 4,
"auto_map": {
"AutoConfig": "configuration_neollm.NeoLLMConfig",
"AutoModel": "modeling_neollm.NeoLLMModel",
"AutoModelForCausalLM": "modeling_neollm.NeoLLMForCausalLM"
},
"bos_token_id": 1,
"cce_loss_impl": "cce_kahan_full_c",
"dagformer": {
"input_source": "tokens",
"local_dim": 128,
"predictor_dim": 256,
"predictor_heads": 4,
"predictor_layers": 2
},
"directional_routing_k": 4,
"directional_routing_temp": 3.0,
"dropout_rate": 0.1,
"dtype": "bfloat16",
"dynamics_metrics_enabled": true,
"dynamics_sample_tokens": 64,
"eos_token_id": 7,
"fan_ratio": 0.125,
"fan_ratio_ffn": 0.0625,
"generator_d_seed": 128,
"generator_k": 3,
"generator_krank": 64,
"generator_num_knots": 16,
"generator_num_modes": 8,
"generator_spline_degree": 2,
"head_dim": 64,
"hidden_act": "xielu",
"hidden_size": 512,
"hola_chunk_size": 256,
"hola_logit_bias_init": -2.0,
"hola_memory_size": 64,
"hola_rms_gamma_init": 1.0,
"hola_score_eps": 1e-06,
"iha_global_layers_use_iha": false,
"iha_local_global_pattern": "LLLLG",
"iha_num_pseudo_heads": 2,
"iha_sliding_window": null,
"initializer_range": 0.02,
"intermediate_size": 1536,
"jtok_aux_loss_weight": 0.0001,
"jtok_kernel_backend": "triton",
"jtok_norm_eps": 1e-06,
"jtok_num_experts": 5,
"jtok_num_modes": 4,
"jtok_top_k": 2,
"laurel_lr_rank": 32,
"liger_loss_accum_dtype": "float32",
"lucid_attention_eps": 1e-06,
"max_position_embeddings": 512,
"mea_component_key_value_heads": 4,
"mea_groupnorm_eps": 1e-06,
"meap_exclude_last": true,
"meap_implementation": "triton",
"meap_mask_ratio": 0.15,
"meap_mask_token_id": 16,
"meap_seed": 42,
"mile_loss_gamma": 1.0,
"model_type": "neollm",
"momentum_gamma": 0.1,
"mu_loss_lambda": 0.0001,
"nextlat_dynamics_proj_factor": 1.0,
"nextlat_horizon": 1,
"nextlat_kl_weight": 1.0,
"nextlat_mse_weight": 1.0,
"nitp_loss_weight": 1.0,
"nitp_projector_intermediate_size": 2048,
"nitp_target_layer": 2,
"nitp_temporal_apply_loss": true,
"nitp_temporal_dynamics_weight": 1.0,
"nitp_temporal_horizon": 4,
"nitp_temporal_identification_weight": 0.1,
"nitp_temporal_intermediate_size": 1024,
"nitp_temporal_similarity_temperature": 0.1,
"nitp_temporal_target_temperature": 0.5,
"ntp_loss_backend": "cce",
"num_attention_heads": 8,
"num_hidden_layers": 12,
"num_key_value_heads": 4,
"num_mem_heads": 4,
"pad_token_id": 0,
"partial_rotary_factor": 0.25,
"polynorm_exclusive": false,
"repo_d_p": 64,
"repo_goat_num_frequencies": 3,
"repo_goat_sink_decay": 4.0,
"repo_grape_circular_prior_strength": 0.01,
"repo_start_layer": 4,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"partial_rotary_factor": 0.25,
"rope_theta": 10000.0,
"rope_type": "default"
},
"rope_theta": 10000.0,
"siamese_attn_x_scale_init": 1.0,
"siamese_depth_scaling": true,
"siamese_normalized_input": true,
"siamesenorm_kernel_backend": "reference",
"stack_d_model": 24,
"stack_entropy_loss_weight": 0.001,
"stack_memory_cache_size": 2048,
"stack_slots": 16,
"tie_word_embeddings": false,
"transformers_version": "5.21.1",
"tweo_eps": 1e-06,
"tweo_loss_weight": 0.01,
"tweo_power": 4.0,
"tweo_tau": 3.0,
"use_affine_scaled_attention": false,
"use_attn_res": false,
"use_cache": false,
"use_copy_paste": true,
"use_dagformer": false,
"use_directional_routing": false,
"use_embedding_input_norm": true,
"use_embedding_multipliers": false,
"use_fan_residual": false,
"use_gpas": false,
"use_hadamard_o_proj": true,
"use_hola_memory": false,
"use_iha": false,
"use_jtok": false,
"use_jtokm": false,
"use_laurel": false,
"use_laurel_lr": false,
"use_laurel_rw": false,
"use_learnable_multipliers": true,
"use_liger_kernel": false,
"use_lns": true,
"use_lucid_attention": false,
"use_mea_attention": false,
"use_meap": true,
"use_mile_loss": true,
"use_momentum_attention": true,
"use_mu_loss": true,
"use_nextlat": false,
"use_nitp": true,
"use_nitp_temporal": false,
"use_perplexity_metrics": true,
"use_repo": true,
"use_repo_goat_prior": false,
"use_repo_grape": true,
"use_siamesenorm": true,
"use_spelling_bee_embeddings": false,
"use_stack_memory": false,
"use_token_generator": true,
"use_topk_metrics": true,
"use_tweo": true,
"use_vha_attention": false,
"use_xsa": true,
"vocab_size": 64402,
"xsa_eps": 1e-06
}
|