config.json · marsnext/Phobos-1 at main

Phobos-1 / config.json

Release MarsNext Phobos-1 (100% 35-Dataset Base Step 33977 + 60-Book 50/50 Replay Post-Trained)

07d2bfc verified 3 days ago

536 Bytes

{
  "architectures": [
    "PhobosScratchLM"
  ],
  "model_type": "phobos_cortical_moe",
  "organization": "MarsNext",
  "vocab_size": 32768,
  "d_model": 512,
  "num_layers": 8,
  "num_heads": 8,
  "num_kv_heads": 4,
  "max_seq_len": 2048,
  "shared_mlp_neurons": 1024,
  "leaf_mlp_neurons": 256,
  "num_branches": 4,
  "leaves_per_layer": [
    16,
    16,
    16,
    16,
    16,
    16,
    16,
    16
  ],
  "top_k_leaves": 2,
  "total_parameters": 96256569,
  "active_parameters_per_token": 52216377,
  "torch_dtype": "bfloat16"
}

	{
	"architectures": [
	"PhobosScratchLM"
	],
	"model_type": "phobos_cortical_moe",
	"organization": "MarsNext",
	"vocab_size": 32768,
	"d_model": 512,
	"num_layers": 8,
	"num_heads": 8,
	"num_kv_heads": 4,
	"max_seq_len": 2048,
	"shared_mlp_neurons": 1024,
	"leaf_mlp_neurons": 256,
	"num_branches": 4,
	"leaves_per_layer": [
	16,
	16,
	16,
	16,
	16,
	16,
	16,
	16
	],
	"top_k_leaves": 2,
	"total_parameters": 96256569,
	"active_parameters_per_token": 52216377,
	"torch_dtype": "bfloat16"
	}