MODAR / models /modar-mp20 /config.json
Coooolin's picture
Release MODAR weights and data without source code
1de13d1 verified
Raw History Blame Contribute Delete
1.52 kB
{
"model": {
"n_layer": 12,
"n_encoder_layer": 0,
"n_head": 12,
"n_embd": 768,
"dropout": 0.1,
"block_size": 1024,
"arch": "v2",
"spectra_slots": 16,
"spectra_grid": 1501,
"rope_base": 10000.0,
"cond_adanorm": true,
"cond_drop": 0.1,
"coord_value_emb": true,
"coord_value_freqs": 16
},
"model_type": "decoder",
"init_from": "external/realpxrd-pretraining/checkpoint_last.pt",
"data": {
"data_dir": "data/mp20",
"condition_set": "mix",
"use_lattice": "mix",
"target_mode": "full",
"block_size": 1024,
"max_records": null,
"rep": "v2",
"spectra_slots": 16,
"spectra_grid_min": 5.0,
"spectra_grid_max": 80.0,
"spectra_grid_step": 0.05,
"spectra_sigma": 0.15,
"mix_condition_sets": [
"pxrd",
"pxrd_element",
"pxrd_xrf",
"pxrd_neutron",
"pxrd_neutron_xrf",
"pxrd_neutron_xrf_xps",
"pxrd_formula"
]
},
"train": {
"batch_size": 32,
"grad_accum_steps": 1,
"lr": 3e-05,
"weight_decay": 0.1,
"max_steps": 6000,
"warmup_steps": 200,
"grad_clip": 1.0,
"log_every": 50,
"eval_every": 250,
"save_every": 500,
"num_workers": 4,
"seed": 42,
"out_dir": "models/modar-mp20",
"device": "cuda",
"amp": "bf16"
},
"world_size": 1,
"vocab_size": 2580,
"n_params": 131672960,
"dataset": {
"train_kept": 27136,
"train_skipped_too_long": 0,
"valid_kept": 9047,
"valid_skipped_too_long": 0
}
}