config.json

{
  "vocab_size": 256,
  "max_seq_len": 129,
  "rope_scale": 1.0,
  "pad_token_id": 0,
  "d_model": 2080,
  "n_layers": 34,
  "n_heads": 16,
  "n_kv_heads": 4,
  "d_head": 130,
  "n_protofilaments": 13,
  "map_hidden_dim": 64,
  "tau_init": 1.0,
  "tau_min": 0.01,
  "tau_max": 10.0,
  "dt": 1.0,
  "polarity_mode": "scalar",
  "polarity_rank": 16,
  "gamma_init": 0.1,
  "n_global_heads": 8,
  "attention_layers": null,
  "gtp_period": 256,
  "n_time_scales": 5,
  "resonance_freqs": [
    0.01,
    0.05623413251903491,
    0.31622776601683794,
    1.7782794100389228,
    10.0
  ],
  "gwtb_compression_ratio": 8,
  "gwtb_n_heads": 1,
  "gwtb_broadcast_init": 0.01,
  "use_gwtb": false,
  "use_global_coherence": false,
  "workspace_iterations": 1,
  "gwtb_per_block": false,
  "core_iterations": 1,
  "core_iter_gate_init": 0.0,
  "signed_decay": false,
  "signed_decay_init": "stock",
  "selective_decay": false,
  "selective_decay_mode": "tanh",
  "selective_decay_snap": false,
  "selective_decay_ste": false,
  "liquid_step_ladder": false,
  "liquid_step_kappa": 1.0,
  "use_chunkwise_scan": false,
  "chunkwise_scan_size": 64,
  "use_householder_transition": false,
  "householder_rank": 2,
  "use_deltaproduct_transition": false,
  "deltaproduct_rank": 2,
  "deltaproduct_scale": 0.1,
  "ffn_swiglu": true,
  "ffn_expansion": 2.6666666666666665,
  "qk_norm": true,
  "scaled_residual_init": true,
  "use_lateral_coupling": true,
  "use_map_gate": true,
  "stack_iterations": 1,
  "use_competitive_gwtb": false,
  "n_competitive_bids": 3,
  "competitive_hard_winner": false,
  "competitive_score_noise": 0.5,
  "competitive_ortho_weight": 0.01,
  "gwtb_external_bids": false,
  "gwtb_external_bid_gate_init": 0.0,
  "gwtb_dynamic_bandwidth": false,
  "gwtb_bandwidth_gate_bias": 4.0,
  "gwtb_bandwidth_threshold": 0.1,
  "gwtb_bandwidth_hard_mask": false,
  "use_top_down": false,
  "top_down_gate_init": 0.0,
  "top_down_goal": false,
  "top_down_to_gwtb": false,
  "top_down_gwtb_gate_init": 0.0,
  "gwtb_memory_bid": false,
  "gwtb_memory_bid_dim": 64,
  "gwtb_memory_bid_heads": 1,
  "gwtb_memory_bid_decay": 0.95,
  "gwtb_memory_bid_gate_init": 0.0,
  "use_graceful_degradation": true,
  "coherence_sparsity": 0.1,
  "coherence_heads": 4,
  "use_decay_wm": false,
  "wm_decay_rate_init": 0.99,
  "compute_skip_threshold": 0.0,
  "dropout": 0.0,
  "attention_dropout": 0.0,
  "tie_embeddings": true,
  "use_predictive_coding": false,
  "predictive_loss_weight": 0.05,
  "direct_target_max_len": 16,
  "use_position_free_attention": false,
  "h_prev_position_weight": 0.05,
  "keep_relative_bias": true,
  "position_free_mode": "hybrid",
  "use_causal_head": false,
  "use_self_monitor_head": false,
  "use_mtp_heads": false,
  "mtp_lookahead": 3,
  "mtp_loss_weight": 0.1,
  "enable_speculative_decoding": false,
  "dynamic_scale_gates": true,
  "scale_gate_init_bias": 2.0,
  "scale_gate_active_threshold": 0.5,
  "scale_gate_skip_threshold": 0.0,
  "sparse_resonance_kernel": false,
  "sparse_resonance_top_k": 1,
  "scale_gate_period": 1,
  "use_hebbian": false,
  "hebbian_lr": 0.0001,
  "hebbian_lavi_gate": true,
  "use_hebbian_refactor": false,
  "hebbian_base_lr": 0.01,
  "hebbian_window": 32,
  "hebbian_grad_frac_cap": 0.05,
  "hebbian_refactor_mode": "weak",
  "use_world_model": false,
  "world_model_loss_weight": 0.01,
  "world_model_hidden_ratio": 0.5,
  "world_model_proj_ratio": 0.5,
  "world_model_ema_decay": 0.99,
  "world_model_use_ema_target": true,
  "world_model_warmup_steps": 1000,
  "use_hamiltonian_world_model": false,
  "hamiltonian_loss_weight": 0.01,
  "hamiltonian_phase_dim": 32,
  "hamiltonian_hidden": 64,
  "hamiltonian_dt": 0.1,
  "hamiltonian_condition_on_context": true,
  "hamiltonian_context_dim": 32,
  "causal_check_method": "cosine",
  "causal_check_window": 5,
  "causal_check_threshold": 0.3,
  "use_rhythm": false,
  "rhythm_scale_init": 0.1,
  "global_rhythm": false,
  "fast_weight_core": false,
  "fast_weight_rank": 16,
  "fast_weight_decay": 0.99,
  "d_proto": 160,
  "d_proto_total": 2080,
  "d_ff": 5632
}