onner / config.json
ReskiGaming's picture
Upload onner model with OnnerReasearch architecture
d9c1356 verified
Raw History Blame Contribute Delete
1.14 kB
{
"architecture_name": "OnnerReasearch",
"model_name": "onner",
"dataset_name": "indonesian-nlp/wikipedia-id",
"vocab_size": 28000,
"tokenizer_train_samples": 50000,
"tokenizer_path": "onner_tokenizer.json",
"cache_docs": 50000,
"full_token_cache_path": "onner_tokens_full.pt",
"train_token_cache_path": "onner_tokens_train.pt",
"val_token_cache_path": "onner_tokens_val.pt",
"val_ratio": 0.02,
"seq_len": 512,
"d_model": 768,
"n_layers": 16,
"n_heads": 12,
"n_kv_heads": 4,
"ffn_hidden": 2048,
"dropout": 0.0,
"rope_base": 10000.0,
"mtp_n": 4,
"mtp_loss_weights": [
1.0,
0.3,
0.2,
0.1
],
"micro_batch_size": 8,
"grad_accum_steps": 8,
"max_steps": 500,
"lr": 0.0003,
"min_lr": 3e-05,
"warmup_steps": 50,
"weight_decay": 0.1,
"grad_clip": 1.0,
"eval_interval": 100,
"eval_steps": 20,
"save_interval": 500,
"checkpoint_dir": "onner_checkpoints",
"dataloader_workers": 0,
"seed": 42,
"parameter_count": 123964416,
"parameter_count_millions": 123.964416,
"model_type": "onner",
"architecture": "OnnerReasearch",
"library_name": "custom-pytorch"
}