freococo's picture
Release MyTranslation_300M_Base (298.76M inf / 9x RTX 5090, Val PPL 3.20 + custom_code, code & logs)
3bcdbab verified
Raw History Blame Contribute Delete
803 Bytes
{
"architectures": [
"MyTranslationForConditionalGeneration"
],
"auto_map": {
"AutoConfig": "configuration_mytranslation.MyTranslationConfig",
"AutoModelForSeq2SeqLM": "modeling_mytranslation.MyTranslationForConditionalGeneration"
},
"model_type": "mytranslation",
"is_encoder_decoder": true,
"tie_word_embeddings": true,
"vocab_size": 65536,
"d_model": 1024,
"n_heads": 16,
"n_kv_heads": 4,
"enc_layers": 12,
"dec_layers": 6,
"ffn_dim": 3072,
"max_seq_len": 2048,
"rope_theta": 50000.0,
"unk_token_id": 0,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 3,
"stage1_final_val_nll": 1.4316,
"stage1_final_val_ppl": 4.19,
"val_nll": 1.164,
"val_ppl": 3.2,
"en2my_val_nll": 1.4829,
"my2en_val_nll": 0.8901,
"torch_dtype": "bfloat16"
}