cl-pncm-tiny-124m / config.json
entity2260's picture
Upload trained C-LPNCM Next-Concept Transformer model
3fcc978 verified
Raw History Blame Contribute Delete
1.64 kB
{
"architectures": [
"CLPNCMForNextConceptModeling"
],
"model_type": "cl_pncm",
"auto_map": {
"AutoConfig": "modeling_cl_pncm.CLPNCMHuggingFaceConfig",
"AutoModel": "modeling_cl_pncm.CLPNCMForNextConceptModeling"
},
"vocab_size": 8192,
"pad_token_id": 0,
"bos_token_id": 1,
"eos_token_id": 2,
"mask_token_id": 3,
"abstain_token_id": 4,
"d_enc": 512,
"d_core": 768,
"d_dec": 512,
"mlp_ratio": 2.6667,
"enc_layers": 4,
"enc_heads": 8,
"enc_kv_heads": 2,
"local_window_size": 128,
"entropy_split_threshold": 2.4,
"min_patch_len": 1,
"max_patch_len": 8,
"n_coarse_codes": 4096,
"vq_ema_decay": 0.99,
"vq_commitment_weight": 0.25,
"fsq_levels": 3,
"fsq_bins": [
8,
8,
8,
6,
5
],
"core_layers": 8,
"core_heads": 12,
"core_kv_heads": 4,
"policy_temp": 1.0,
"tau_high": 0.85,
"tau_low": 0.4,
"deliberative_beams": 4,
"loop_layers": 3,
"loop_heads": 8,
"max_loops": 6,
"halt_tol": 0.001,
"share_loop_weights": true,
"use_xsa": true,
"use_attn_gate": true,
"deep_supervision_weighting": "exponential",
"dec_layers": 4,
"dec_heads": 8,
"pointer_dropout": 0.5,
"boundary_conf_threshold": 0.75,
"stage": 1,
"gamma_copy": 0.5,
"alpha_loop": 1.0,
"eta_contract": 0.2,
"contraction_rho": 0.7,
"grpo_group_size": 4,
"lambda_brier": 1.0,
"mu_ponder": 0.02,
"beta_kl": 0.01,
"c_abstain": 0.15,
"learning_rate": 0.0003,
"warmup_steps": 200,
"num_steps": 5000,
"batch_size": 32,
"micro_batch_size": 4,
"max_grad_norm": 1.0,
"adam_beta2": 0.95,
"weight_decay": 0.01,
"seed": 42,
"amp_dtype": "bf16"
}