| # Nexus Coder Configuration - XLarge (~30B/3B) v0.3 | |
| # Research-only. Requires 64+ H100 80GB GPUs. | |
| # Author: Hieu Louis (2026) | |
| model: | |
| name: "Nexus Coder XLarge" | |
| agent_name: "Nexus" | |
| author: "Hieu Louis" | |
| version: "0.3.0-xlarge" | |
| github: "mhieuhonda" | |
| year: "2026" | |
| architecture: | |
| vocab_size: 64000 | |
| hidden_size: 4096 | |
| num_hidden_layers: 24 | |
| num_attention_heads: 32 | |
| num_kv_heads: 8 | |
| head_dim: 128 | |
| intermediate_size: 11264 | |
| hidden_act: "silu" | |
| norm_type: "rmsnorm" | |
| moe: | |
| num_experts: 48 | |
| num_active_experts: 4 | |
| router_aux_loss_coef: 0.001 | |
| context: | |
| max_position_embeddings: 65536 # 64k tokens | |
| rotary_emb_base: 10000.0 | |
| rope_scaling_type: "dynamic" # NTK-aware for 2× context extension | |
| rope_scaling_factor: 2.0 | |
| attention: | |
| use_flash_attention: true | |
| use_flash_attention_2: true # recommended at this scale | |
| use_alibi: false | |
| use_sliding_window: true | |
| sliding_window_size: 8192 | |
| use_qk_norm: true | |
| mlp_parallel: true | |
| compute: | |
| use_kv_cache: true | |
| kv_cache_quantization: "int8" # saves KV cache memory at long context | |
| gradient_checkpointing: true # essential at this scale | |
| tensor_parallel_size: 4 | |
| pipeline_parallel_size: 1 | |
| expert_parallel_size: 4 | |
| sequence_parallel: false | |
| params: | |
| total: "~30B" | |
| active: "~3B" | |
| expert_utilization: "8.3%" | |
| estimated_disk_mb_fp16: 60000 | |
| estimated_disk_mb_int8: 30000 | |
| estimated_disk_mb_int4: 15000 | |
| training: | |
| learning_rate: 2.0e-4 | |
| weight_decay: 0.01 | |
| warmup_steps: 500 | |
| max_steps: 10000 | |
| per_device_batch_size: 1 | |
| gradient_accumulation_steps: 32 | |
| logging_steps: 10 | |
| save_steps: 1000 | |
| max_grad_norm: 1.0 | |
| seed: 42 | |
| use_amp: true | |
| use_deepspeed: true | |
| deepspeed_config: "configs/ds_config_zero3.json" | |
| inference: | |
| max_new_tokens: 500 | |
| temperature: 0.7 | |
| top_k: 50 | |
| top_p: 0.9 | |
| do_sample: true | |
| personality: | |
| type: "humorous" | |
| language: "bilingual" | |
| environment: | |
| python_version: "3.12.13" | |
| pytorch_version: ">=2.0" | |
| cuda_required: true | |
| min_gpu_memory_gb: 80 | |
| recommended_gpus: "64+ H100 80GB" | |