{
    "profile": null,
    "name": "slimpajama_moe_no_attmoe_660M_standardlb_zloss",
    "reset": 0,
    "log": "tb",
    "save_interval": "20000",
    "wandb_save_interval": "None",
    "seed": "none",
    "gpu": "0,1,2,3",
    "keep_alive": 0,
    "sweep_id_for_grid_search": 0,
    "restore": "",
    "wandb_bug_workaround": 0,
    "wandb_sync_checkpoints": 0,
    "batch_size": 64,
    "lr": 0.00025,
    "min_lr_multiplier": 0.1,
    "wd": 0.01,
    "lr_warmup": 4000,
    "test_interval": 20000,
    "n_microbatch": "None",
    "per_device_batch_size": "16",
    "lr_sched.steps": "",
    "lr_sched.gamma": 0.1,
    "lr_sched.type": "cos",
    "length_bucketed_sampling": 0,
    "grad_clip": "0.25",
    "test_batch_size": "None",
    "val_log_details": 0,
    "reg_scales": "",
    "reg_lin_decay": "",
    "reg": 1.0,
    "optimizer": "adamw",
    "adam.betas": "0.9,0.999",
    "adam.eps": 1e-08,
    "stop_after": "400000",
    "amp": 1,
    "bfloat16": 1,
    "nan_detect": 0,
    "max_length_per_batch": "none",
    "log_grad_norms": 0,
    "speedtest": "none",
    "dump_logs": 0,
    "debug_plot_interval": "none",
    "lm.trafo.context_blocks": 0,
    "lm.trafo.test_context_blocks": "none",
    "lm.trafo.same_length_eval": 0,
    "lm.trafo.same_length": 0,
    "lm.trafo.last_layer_context": 0,
    "lm.trafo.xl_init": 0,
    "lm.trafo.norm_input": 0,
    "rope.rotate_fraction": 0.5,
    "rope.base": 10000.0,
    "pkm.n_heads": 8,
    "moe.n_experts": 66,
    "moe.expert_size": 256,
    "moe.selection_mode": "gate",
    "moe.perplexity_reg": 0.01,
    "moe.perplexity_reg_mode": "standard_with_zloss",
    "moe.att.perplexity_reg_mode": "none",
    "moe.activation_after_topk": 0,
    "moe.att.expert_size": 256,
    "moe.bias": 0,
    "moe.sel_bias": 0,
    "moe.dropout_factor": 1.0,
    "moe.drop_expert": 0.0,
    "moe.sync_distributed": 1,
    "moe.init_scale": 1.0,
    "moe.att.mla_attention": 0,
    "moe.att.n_experts": 4,
    "moe.att.enable": 0,
    "moe.att.q_expert": 1,
    "moe.att.k_expert": 1,
    "moe.att.v_expert": 1,
    "moe.att.o_expert": 1,
    "moe.att.k": 2,
    "moe.att.v_size": "none",
    "moe.att.same_sel": 0,
    "moe.att.expert_dropout": "none",
    "moe.att.selection_mode": "sigmoid",
    "moe.att.perplexity_reg": "none",
    "moe.att.drop_expert": "none",
    "moe.att.separate_kq_sel": 0,
    "moe.att.norm_init": 0,
    "moe.att.dropout": 0.0,
    "moe.att.selection_dropout": 0.0,
    "moe.nonorm": 0,
    "moa.cvloss": 0.0,
    "moa.switchloss": 0.0,
    "moa.zloss": 0.0,
    "moa.miloss": 0.0,
    "sut.sample_topk": 0,
    "sut.max_relative_positions": 64,
    "sut.drop_gate": 0.0,
    "moe.selection_dropout": 0.0,
    "moe.layer_std_constant": 2.0,
    "transformer.universal.group_size": 18,
    "transformer.universal.group_type": "abab",
    "transformer.embedding_scale": "none",
    "transformer.topk_value": 32,
    "transformer.activation": "relu",
    "transformer.p_drop_layer": 0.0,
    "transformer.head_projection_size": "128",
    "transformer.act_loss": 0.0,
    "transformer.plot_head_details": 0,
    "lm.trafo.force_out_norm": 0,
    "plot.n_steps": -128,
    "dump_validation_plots": "",
    "details_log_interval": "500",
    "lm.state_drop_probability": 0.0,
    "lm.unroll": 1024,
    "lm.unroll_eval": "none",
    "lm.example_context": 100,
    "lm.example_window": 40,
    "lm.eval.blimp.batch_mul": 16,
    "lm.eval.enabled": 0,
    "lm.eval.lambada.enabled": 0,
    "lm.eval.cbt.batch_mul": 1,
    "lm.eval.cbt.length_limit": "none",
    "lm.eval.cbt.enabled": 0,
    "lm.eval.cbt.end_only": 0,
    "lm.eval.blimp.enabled": 0,
    "lm.eval.hellaswag.enabled": 0,
    "lm.eval.hellaswag.batch_mul": 16,
    "lm.eval.piqa.enabled": 0,
    "lm.eval.piqa.batch_mul": 16,
    "lm.eval.ai2arc.enabled": 0,
    "lm.eval.ai2arc.batch_mul": 4,
    "lm.eval.boolq.enabled": 0,
    "lm.eval.mmlu.enabled": 0,
    "lm.eval.openbookqa.enabled": 0,
    "lm.eval.race.enabled": 0,
    "lm.eval.siqa.enabled": 0,
    "lm.eval.winogrande.enabled": 0,
    "lm.eval.commonsenseqa.enabled": 0,
    "sentencepiece.n_pieces": 8000,
    "lmds.valid_ratio": 0.005,
    "thestack.languages": "python,html,c++,rust,javascript,haskell,scala,assembly",
    "state_size": 1024,
    "task": "slimpajama_large_transformer",
    "dropout": 0.0,
    "embedding_size": "none",
    "transformer.n_heads": 4,
    "transformer.variant": "preln_moe",
    "transformer.ff_multiplier": 2.0,
    "transformer.encoder_n_layers": 18,
    "transformer.attention_dropout": 0.0,
    "load_pretrained_model": null,
    "test_pretrained": 1,
    "train_baseline": 0,
    "test_only": 0,
    "fs_cache_pattern": "*"
}