architecture.json

{"expert_start": 0, "expert_end": 3, "output_index": 8, "num_experts": 3, "num_output_versions": 2, "vocab_size": 50259, "n_embd": 768, "n_head": 12, "n_layer": 9, "n_positions": 1024}